Merge pull request #9865 from wy65701436/quota-event

add quota exceed event imple

src/core/middlewares/countquota/handler.go

@@ -59,6 +59,7 @@ func (h *countQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
 	if err := interceptor.HandleRequest(req); err != nil {
 		log.Warningf("Error occurred when to handle request in count quota handler: %v", err)
 		if _, ok := err.(quota.Errors); ok {
+			util.FireQuotaEvent(req, 1, err.Error())
 			http.Error(rw, util.MarshalError("DENIED", fmt.Sprintf("Quota exceeded when processing the request of %v", err)), http.StatusForbidden)
 			return
 		}

src/core/middlewares/sizequota/handler.go

@@ -59,6 +59,7 @@ func (h *sizeQuotaHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request)
 	if err := interceptor.HandleRequest(req); err != nil {
 		log.Warningf("Error occurred when to handle request in size quota handler: %v", err)
 		if _, ok := err.(quota.Errors); ok {
+			util.FireQuotaEvent(req, 1, err.Error())
 			http.Error(rw, util.MarshalError("DENIED", fmt.Sprintf("Quota exceeded when processing the request of %v", err)), http.StatusForbidden)
 			return
 		}

src/core/middlewares/util/util.go

@@ -37,6 +37,8 @@ import (
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/core/config"
+	"github.com/goharbor/harbor/src/core/filter"
+	notifierEvt "github.com/goharbor/harbor/src/core/notifier/event"
 	"github.com/goharbor/harbor/src/core/promgr"
 	"github.com/goharbor/harbor/src/pkg/scan/vuln"
 	"github.com/goharbor/harbor/src/pkg/scan/whitelist"
@@ -563,6 +565,49 @@ func ParseManifestInfoFromPath(req *http.Request) (*ManifestInfo, error) {
 	return info, nil
 }
 
+// FireQuotaEvent ...
+func FireQuotaEvent(req *http.Request, level int, msg string) {
+	go func() {
+		info, err := ParseManifestInfoFromReq(req)
+		if err != nil {
+			log.Errorf("Quota exceed event: failed to get manifest from request: %v", err)
+			return
+		}
+		pm, err := filter.GetProjectManager(req)
+		if err != nil {
+			log.Errorf("Quota exceed event: failed to get project manager: %v", err)
+			return
+		}
+		project, err := pm.Get(info.ProjectID)
+		if err != nil {
+			log.Errorf(fmt.Sprintf("Quota exceed event: failed to get the project %d", info.ProjectID), err)
+			return
+		}
+		if project == nil {
+			log.Errorf(fmt.Sprintf("Quota exceed event: no project found %d", info.ProjectID), err)
+			return
+		}
+
+		evt := &notifierEvt.Event{}
+		quotaMetadata := &notifierEvt.QuotaMetaData{
+			Project:  project,
+			Tag:      info.Tag,
+			Digest:   info.Digest,
+			RepoName: info.Repository,
+			Level:    level,
+			Msg:      msg,
+			OccurAt:  time.Now(),
+		}
+		if err := evt.Build(quotaMetadata); err == nil {
+			if err := evt.Publish(); err != nil {
+				log.Errorf("failed to publish quota event: %v", err)
+			}
+		} else {
+			log.Errorf("failed to build quota event metadata: %v", err)
+		}
+	}()
+}
+
 func getProjectVulnSeverity(project *models.Project) vuln.Severity {
 	mp := map[string]vuln.Severity{
 		models.SeverityNegligible: vuln.Negligible,

src/core/notifier/event/event.go

@@ -222,6 +222,48 @@ func (si *ScanImageMetaData) Resolve(evt *Event) error {
 	return nil
 }
 
+// QuotaMetaData defines quota related event data
+type QuotaMetaData struct {
+	Project  *models.Project
+	RepoName string
+	Tag      string
+	Digest   string
+	// used to define the event topic
+	Level int
+	// the msg contains the limitation and current usage of quota
+	Msg     string
+	OccurAt time.Time
+}
+
+// Resolve quota exceed into common image event
+func (q *QuotaMetaData) Resolve(evt *Event) error {
+	var topic string
+	data := &model.QuotaEvent{
+		EventType: notifyModel.EventTypeProjectQuota,
+		Project:   q.Project,
+		Resource: &model.ImgResource{
+			Tag:    q.Tag,
+			Digest: q.Digest,
+		},
+		OccurAt:  q.OccurAt,
+		RepoName: q.RepoName,
+		Msg:      q.Msg,
+	}
+
+	switch q.Level {
+	case 1:
+		topic = model.QuotaExceedTopic
+	case 2:
+		topic = model.QuotaWarningTopic
+	default:
+		return errors.New("not supported quota status")
+	}
+
+	evt.Topic = topic
+	evt.Data = data
+	return nil
+}
+
 // HookMetaData defines hook notification related event data
 type HookMetaData struct {
 	PolicyID  int64

src/core/notifier/handler/notification/quota_handler.go

@@ -0,0 +1,103 @@
+package notification
+
+import (
+	"errors"
+	"fmt"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/common/utils/log"
+	"github.com/goharbor/harbor/src/core/config"
+	"github.com/goharbor/harbor/src/core/notifier/model"
+	notifyModel "github.com/goharbor/harbor/src/core/notifier/model"
+	"github.com/goharbor/harbor/src/pkg/notification"
+)
+
+// QuotaPreprocessHandler preprocess image event data
+type QuotaPreprocessHandler struct {
+}
+
+// Handle ...
+func (qp *QuotaPreprocessHandler) Handle(value interface{}) error {
+	if !config.NotificationEnable() {
+		log.Debug("notification feature is not enabled")
+		return nil
+	}
+
+	quotaEvent, ok := value.(*model.QuotaEvent)
+	if !ok {
+		return errors.New("invalid quota event type")
+	}
+	if quotaEvent == nil {
+		return fmt.Errorf("nil quota event")
+	}
+
+	project, err := config.GlobalProjectMgr.Get(quotaEvent.Project.Name)
+	if err != nil {
+		log.Errorf("failed to get project:%s, error: %v", quotaEvent.Project.Name, err)
+		return err
+	}
+	if project == nil {
+		return fmt.Errorf("project not found of quota event: %s", quotaEvent.Project.Name)
+	}
+	policies, err := notification.PolicyMgr.GetRelatedPolices(project.ProjectID, quotaEvent.EventType)
+	if err != nil {
+		log.Errorf("failed to find policy for %s event: %v", quotaEvent.EventType, err)
+		return err
+	}
+	if len(policies) == 0 {
+		log.Debugf("cannot find policy for %s event: %v", quotaEvent.EventType, quotaEvent)
+		return nil
+	}
+
+	payload, err := constructQuotaPayload(quotaEvent)
+	if err != nil {
+		return err
+	}
+
+	err = sendHookWithPolicies(policies, payload, quotaEvent.EventType)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// IsStateful ...
+func (qp *QuotaPreprocessHandler) IsStateful() bool {
+	return false
+}
+
+func constructQuotaPayload(event *model.QuotaEvent) (*model.Payload, error) {
+	repoName := event.RepoName
+	if repoName == "" {
+		return nil, fmt.Errorf("invalid %s event with empty repo name", event.EventType)
+	}
+
+	repoType := models.ProjectPrivate
+	if event.Project.IsPublic() {
+		repoType = models.ProjectPublic
+	}
+
+	imageName := getNameFromImgRepoFullName(repoName)
+	quotaCustom := make(map[string]string)
+	quotaCustom["Details"] = event.Msg
+
+	payload := &notifyModel.Payload{
+		Type:    event.EventType,
+		OccurAt: event.OccurAt.Unix(),
+		EventData: &notifyModel.EventData{
+			Repository: &notifyModel.Repository{
+				Name:         imageName,
+				Namespace:    event.Project.Name,
+				RepoFullName: repoName,
+				RepoType:     repoType,
+			},
+			Custom: quotaCustom,
+		},
+	}
+	resource := &notifyModel.Resource{
+		Tag:    event.Resource.Tag,
+		Digest: event.Resource.Digest,
+	}
+	payload.EventData.Resources = append(payload.EventData.Resources, resource)
+
+	return payload, nil
+}

src/core/notifier/handler/notification/quota_handler_test.go

@@ -0,0 +1,87 @@
+package notification
+
+import (
+	"testing"
+	"time"
+
+	"github.com/goharbor/harbor/src/common"
+	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/core/config"
+	"github.com/goharbor/harbor/src/core/notifier"
+	"github.com/goharbor/harbor/src/core/notifier/model"
+	"github.com/goharbor/harbor/src/pkg/notification"
+	nm "github.com/goharbor/harbor/src/pkg/notification/model"
+	"github.com/goharbor/harbor/src/pkg/notification/policy"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+// QuotaPreprocessHandlerSuite ...
+type QuotaPreprocessHandlerSuite struct {
+	suite.Suite
+	om  policy.Manager
+	evt *model.QuotaEvent
+}
+
+// TestQuotaPreprocessHandler ...
+func TestQuotaPreprocessHandler(t *testing.T) {
+	suite.Run(t, &QuotaPreprocessHandlerSuite{})
+}
+
+// SetupSuite prepares env for test suite.
+func (suite *QuotaPreprocessHandlerSuite) SetupSuite() {
+	cfg := map[string]interface{}{
+		common.NotificationEnable: true,
+	}
+	config.InitWithSettings(cfg)
+
+	res := &model.ImgResource{
+		Digest: "sha256:abcd",
+		Tag:    "latest",
+	}
+	suite.evt = &model.QuotaEvent{
+		EventType: nm.EventTypeProjectQuota,
+		OccurAt:   time.Now().UTC(),
+		RepoName:  "hello-world",
+		Resource:  res,
+		Project: &models.Project{
+			ProjectID: 1,
+			Name:      "library",
+		},
+		Msg: "this is a testing quota event",
+	}
+
+	suite.om = notification.PolicyMgr
+	mp := &fakedPolicyMgr{}
+	notification.PolicyMgr = mp
+
+	h := &MockHandler{}
+
+	err := notifier.Subscribe(model.WebhookTopic, h)
+	require.NoError(suite.T(), err)
+}
+
+// TearDownSuite ...
+func (suite *QuotaPreprocessHandlerSuite) TearDownSuite() {
+	notification.PolicyMgr = suite.om
+}
+
+// TestHandle ...
+func (suite *QuotaPreprocessHandlerSuite) TestHandle() {
+	handler := &QuotaPreprocessHandler{}
+	err := handler.Handle(suite.evt)
+	suite.NoError(err)
+}
+
+// MockHandler ...
+type MockHandler struct{}
+
+// Handle ...
+func (m *MockHandler) Handle(value interface{}) error {
+	return nil
+}
+
+// IsStateful ...
+func (m *MockHandler) IsStateful() bool {
+	return false
+}

src/core/notifier/model/event.go

@@ -41,6 +41,16 @@ type ScanImageEvent struct {
 	Operator  string
 }
 
+// QuotaEvent is project quota related event data to publish
+type QuotaEvent struct {
+	EventType string
+	Project   *models.Project
+	Resource  *ImgResource
+	OccurAt   time.Time
+	RepoName  string
+	Msg       string
+}
+
 // HookEvent is hook related event data to publish
 type HookEvent struct {
 	PolicyID  int64
@@ -53,14 +63,15 @@ type HookEvent struct {
 type Payload struct {
 	Type      string     `json:"type"`
 	OccurAt   int64      `json:"occur_at"`
-	EventData *EventData `json:"event_data,omitempty"`
 	Operator  string     `json:"operator"`
+	EventData *EventData `json:"event_data,omitempty"`
 }
 
 // EventData of notification event payload
 type EventData struct {
-	Resources  []*Resource `json:"resources"`
+	Resources  []*Resource       `json:"resources"`
-	Repository *Repository `json:"repository"`
+	Repository *Repository       `json:"repository"`
+	Custom     map[string]string `json:"custom_attributes,omitempty"`
 }
 
 // Resource describe infos of resource triggered notification

src/core/notifier/model/topic.go

@@ -18,6 +18,10 @@ const (
 	ScanningFailedTopic = "OnScanningFailed"
 	// ScanningCompletedTopic is topic for scanning completed event
 	ScanningCompletedTopic = "OnScanningCompleted"
+	// QuotaExceedTopic is topic for quota warning event, the usage reaches the warning bar of limitation, like 85%
+	QuotaWarningTopic = "OnQuotaWarning"
+	// QuotaExceedTopic is topic for quota exceeded event
+	QuotaExceedTopic = "OnQuotaExceed"
 
 	// WebhookTopic is topic for sending webhook payload
 	WebhookTopic = "http"

src/core/notifier/topic/topics.go

@@ -19,6 +19,7 @@ func init() {
 		model.DeleteChartTopic:       {&notification.ChartPreprocessHandler{}},
 		model.ScanningCompletedTopic: {&notification.ScanImagePreprocessHandler{}},
 		model.ScanningFailedTopic:    {&notification.ScanImagePreprocessHandler{}},
+		model.QuotaExceedTopic:       {&notification.QuotaPreprocessHandler{}},
 	}
 
 	for t, handlers := range handlersMap {

src/pkg/notification/model/const.go

@@ -11,6 +11,7 @@ const (
 	EventTypeScanningCompleted = "scanningCompleted"
 	EventTypeScanningFailed    = "scanningFailed"
 	EventTypeTestEndpoint      = "testEndpoint"
+	EventTypeProjectQuota      = "projectQuota"
 
 	NotifyTypeHTTP = "http"
 )

src/pkg/notification/notification.go

@@ -42,7 +42,7 @@ func Init() {
 	initSupportedEventType(
 		model.EventTypePushImage, model.EventTypePullImage, model.EventTypeDeleteImage,
 		model.EventTypeUploadChart, model.EventTypeDeleteChart, model.EventTypeDownloadChart,
-		model.EventTypeScanningCompleted, model.EventTypeScanningFailed,
+		model.EventTypeScanningCompleted, model.EventTypeScanningFailed, model.EventTypeProjectQuota,
 	)
 
 	initSupportedNotifyType(model.NotifyTypeHTTP)

src/portal/src/app/shared/shared.const.ts

@@ -109,4 +109,5 @@ export enum WebhookEventTypes {
   PUSH_IMAGE = "pushImage",
   SCANNING_FAILED = "scanningFailed",
   SCANNING_COMPLETED = "scanningCompleted",
+  PROJECT_QUOTA = "projectQuota",
 }