Merge branch 'master' of https://github.com/goharbor/harbor into fix-resource-order

.github/workflows/CI.yml

@@ -27,6 +27,7 @@ jobs:
     runs-on:
       #- self-hosted
       - ubuntu-latest
+    timeout-minutes: 60
     steps:
       - name: Set up Go 1.14
         uses: actions/setup-go@v1
@@ -92,6 +93,7 @@ jobs:
     runs-on:
       #- self-hosted
       - ubuntu-latest
+    timeout-minutes: 60
     steps:
       - name: Set up Go 1.14
         uses: actions/setup-go@v1
@@ -195,6 +197,7 @@ jobs:
     runs-on:
       #- self-hosted
       - ubuntu-latest
+    timeout-minutes: 60
     steps:
       - name: Set up Go 1.14
         uses: actions/setup-go@v1
@@ -253,6 +256,7 @@ jobs:
     runs-on:
       #- self-hosted
       - ubuntu-latest
+    timeout-minutes: 60
     steps:
       - name: Set up Go 1.14
         uses: actions/setup-go@v1
@@ -309,6 +313,7 @@ jobs:
     runs-on:
       #- self-hosted
       - ubuntu-latest
+    timeout-minutes: 60
     steps:
       - uses: actions/setup-node@v1
         with:

Makefile

@@ -90,6 +90,7 @@ GEN_TLS=
 # for docker image tag
 VERSIONTAG=dev
 # for base docker image tag
+PUSHBASEIMAGE=
 BASEIMAGETAG=dev
 BASEIMAGENAMESPACE=goharbor
 # for harbor package name
@@ -422,7 +423,9 @@ build_base_docker:
 	@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
 		echo $$name ; \
 		$(DOCKERBUILD) --pull --no-cache -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) --label base-build-date=$(date +"%Y%m%d") . && \
-		$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) || exit 1; \
+		if [ -n "$(PUSHBASEIMAGE)" ] ; then \
+			$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) || exit 1; \
+		fi ; \
 	done
 
 pull_base_docker:

api/v2.0/swagger.yaml

@@ -605,6 +605,8 @@ paths:
           $ref: '#/responses/403'
         '404':
           $ref: '#/responses/404'
+        '405':
+          $ref: '#/responses/405'
         '409':
           $ref: '#/responses/409'
         '500':

src/common/models/cve_allowlist.go

@@ -14,7 +14,9 @@
 
 package models
 
-import "time"
+import (
+	"time"
+)
 
 // CVEAllowlist defines the data model for a CVE allowlist
 type CVEAllowlist struct {
@@ -38,8 +40,8 @@ func (c *CVEAllowlist) TableName() string {
 }
 
 // CVESet returns the set of CVE id of the items in the allowlist to help filter the vulnerability list
-func (c *CVEAllowlist) CVESet() map[string]struct{} {
-	r := map[string]struct{}{}
+func (c *CVEAllowlist) CVESet() CVESet {
+	r := CVESet{}
 	for _, it := range c.Items {
 		r[it.CVEID] = struct{}{}
 	}
@@ -53,3 +55,13 @@ func (c *CVEAllowlist) IsExpired() bool {
 	}
 	return time.Now().Unix() >= *c.ExpiresAt
 }
+
+// CVESet defines the CVE allowlist with a hash set way for easy query.
+type CVESet map[string]struct{}
+
+// Contains checks whether the specified CVE is in the set or not.
+func (cs CVESet) Contains(cve string) bool {
+	_, ok := cs[cve]
+
+	return ok
+}

src/common/models/cve_allowlist_test.go

@@ -15,10 +15,10 @@
 package models
 
 import (
-	"github.com/stretchr/testify/assert"
-	"reflect"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestCVEAllowlist_All(t *testing.T) {
@@ -26,7 +26,7 @@ func TestCVEAllowlist_All(t *testing.T) {
 	now := time.Now().Unix()
 	cases := []struct {
 		input   CVEAllowlist
-		cveset  map[string]struct{}
+		cveset  CVESet
 		expired bool
 	}{
 		{
@@ -35,7 +35,7 @@ func TestCVEAllowlist_All(t *testing.T) {
 				ProjectID: 0,
 				Items:     []CVEAllowlistItem{},
 			},
-			cveset:  map[string]struct{}{},
+			cveset:  CVESet{},
 			expired: false,
 		},
 		{
@@ -45,7 +45,7 @@ func TestCVEAllowlist_All(t *testing.T) {
 				Items:     []CVEAllowlistItem{},
 				ExpiresAt: &now,
 			},
-			cveset:  map[string]struct{}{},
+			cveset:  CVESet{},
 			expired: true,
 		},
 		{
@@ -58,7 +58,7 @@ func TestCVEAllowlist_All(t *testing.T) {
 				},
 				ExpiresAt: &future,
 			},
-			cveset: map[string]struct{}{
+			cveset: CVESet{
 				"CVE-1999-0067":    {},
 				"CVE-2016-7654321": {},
 			},
@@ -67,6 +67,6 @@ func TestCVEAllowlist_All(t *testing.T) {
 	}
 	for _, c := range cases {
 		assert.Equal(t, c.expired, c.input.IsExpired())
-		assert.True(t, reflect.DeepEqual(c.cveset, c.input.CVESet()))
+		assert.Equal(t, c.cveset, c.input.CVESet())
 	}
 }

src/controller/event/handler/init.go

@@ -28,6 +28,7 @@ func init() {
 	notifier.Subscribe(event.TopicScanningCompleted, &scan.Handler{})
 	notifier.Subscribe(event.TopicDeleteArtifact, &scan.DelArtHandler{})
 	notifier.Subscribe(event.TopicReplication, &artifact.ReplicationHandler{})
+	notifier.Subscribe(event.TopicTagRetention, &artifact.RetentionHandler{RetentionController: artifact.DefaultRetentionControllerFunc})
 
 	// replication
 	notifier.Subscribe(event.TopicPushArtifact, &replication.Handler{})

src/controller/event/handler/webhook/artifact/replication.go

@@ -8,6 +8,7 @@ import (
 	commonModels "github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/controller/event"
 	"github.com/goharbor/harbor/src/controller/event/handler/util"
+	ctlModel "github.com/goharbor/harbor/src/controller/event/model"
 	"github.com/goharbor/harbor/src/controller/project"
 	"github.com/goharbor/harbor/src/core/config"
 	"github.com/goharbor/harbor/src/jobservice/job"
@@ -15,7 +16,6 @@ import (
 	"github.com/goharbor/harbor/src/lib/orm"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/pkg/notifier/model"
-	notifyModel "github.com/goharbor/harbor/src/pkg/notifier/model"
 	"github.com/goharbor/harbor/src/replication"
 	rpModel "github.com/goharbor/harbor/src/replication/model"
 )
@@ -120,7 +120,7 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
 	}
 	hostname := strings.Split(extURL, ":")[0]
 
-	remoteRes := &model.ReplicationResource{
+	remoteRes := &ctlModel.ReplicationResource{
 		RegistryName: remoteRegistry.Name,
 		RegistryType: string(remoteRegistry.Type),
 		Endpoint:     remoteRegistry.URL,
@@ -131,18 +131,18 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
 	if err != nil {
 		log.Errorf("Error while reading external endpoint: %v", err)
 	}
-	localRes := &model.ReplicationResource{
+	localRes := &ctlModel.ReplicationResource{
 		RegistryType: string(rpModel.RegistryTypeHarbor),
 		Endpoint:     ext,
 		Namespace:    destNamespace,
 	}
 
-	payload := &notifyModel.Payload{
+	payload := &model.Payload{
 		Type:     event.EventType,
 		OccurAt:  event.OccurAt.Unix(),
 		Operator: string(execution.Trigger),
 		EventData: &model.EventData{
-			Replication: &model.Replication{
+			Replication: &ctlModel.Replication{
 				HarborHostname:     hostname,
 				JobStatus:          event.Status,
 				Description:        rpPolicy.Description,
@@ -174,20 +174,20 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
 	}
 
 	if event.Status == string(job.SuccessStatus) {
-		succeedArtifact := &model.ArtifactInfo{
+		succeedArtifact := &ctlModel.ArtifactInfo{
 			Type:       task.ResourceType,
 			Status:     task.Status,
 			NameAndTag: nameAndTag,
 		}
-		payload.EventData.Replication.SuccessfulArtifact = []*model.ArtifactInfo{succeedArtifact}
+		payload.EventData.Replication.SuccessfulArtifact = []*ctlModel.ArtifactInfo{succeedArtifact}
 	}
 	if event.Status == string(job.ErrorStatus) {
-		failedArtifact := &model.ArtifactInfo{
+		failedArtifact := &ctlModel.ArtifactInfo{
 			Type:       task.ResourceType,
 			Status:     task.Status,
 			NameAndTag: nameAndTag,
 		}
-		payload.EventData.Replication.FailedArtifact = []*model.ArtifactInfo{failedArtifact}
+		payload.EventData.Replication.FailedArtifact = []*ctlModel.ArtifactInfo{failedArtifact}
 	}
 
 	prj, err := project.Ctl.GetByName(orm.Context(), prjName, project.Metadata(true))

src/controller/event/handler/webhook/artifact/retention.go

@@ -0,0 +1,160 @@
+package artifact
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/goharbor/harbor/src/controller/event"
+	"github.com/goharbor/harbor/src/controller/event/handler/util"
+	evtModel "github.com/goharbor/harbor/src/controller/event/model"
+	"github.com/goharbor/harbor/src/core/api"
+	"github.com/goharbor/harbor/src/core/config"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/pkg/notification"
+	"github.com/goharbor/harbor/src/pkg/notifier/model"
+	"github.com/goharbor/harbor/src/pkg/retention"
+)
+
+// RetentionHandler preprocess tag retention event data
+type RetentionHandler struct {
+	RetentionController func() retention.APIController
+}
+
+// DefaultRetentionControllerFunc ...
+var DefaultRetentionControllerFunc = NewRetentionController
+
+// NewRetentionController ...
+func NewRetentionController() retention.APIController {
+	return api.GetRetentionController()
+}
+
+// Handle ...
+func (r *RetentionHandler) Handle(value interface{}) error {
+	if !config.NotificationEnable() {
+		log.Debug("notification feature is not enabled")
+		return nil
+	}
+
+	trEvent, ok := value.(*event.RetentionEvent)
+	if !ok {
+		return errors.New("invalid tag retention event type")
+	}
+	if trEvent == nil {
+		return errors.New("nil tag retention event")
+	}
+	if len(trEvent.Deleted) == 0 {
+		log.Debugf("empty delete info of retention event")
+		return nil
+	}
+
+	payload, dryRun, project, err := r.constructRetentionPayload(trEvent)
+	if err != nil {
+		return err
+	}
+	// if dry run, do not trigger webhook
+	if dryRun {
+		log.Debugf("retention task %v is dry run", trEvent.TaskID)
+		return nil
+	}
+
+	policies, err := notification.PolicyMgr.GetRelatedPolices(project, trEvent.EventType)
+	if err != nil {
+		log.Errorf("failed to find policy for %s event: %v", trEvent.EventType, err)
+		return err
+	}
+	if len(policies) == 0 {
+		log.Debugf("cannot find policy for %s event: %v", trEvent.EventType, trEvent)
+		return nil
+	}
+	err = util.SendHookWithPolicies(policies, payload, trEvent.EventType)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// IsStateful ...
+func (r *RetentionHandler) IsStateful() bool {
+	return false
+}
+
+func (r *RetentionHandler) constructRetentionPayload(event *event.RetentionEvent) (*model.Payload, bool, int64, error) {
+	task, err := r.RetentionController().GetRetentionExecTask(event.TaskID)
+	if err != nil {
+		log.Errorf("failed to get retention task %d: error: %v", event.TaskID, err)
+		return nil, false, 0, err
+	}
+	if task == nil {
+		return nil, false, 0, fmt.Errorf("task %d not found with retention event", event.TaskID)
+	}
+
+	execution, err := r.RetentionController().GetRetentionExec(task.ExecutionID)
+	if err != nil {
+		log.Errorf("failed to get retention execution %d: error: %v", task.ExecutionID, err)
+		return nil, false, 0, err
+	}
+	if execution == nil {
+		return nil, false, 0, fmt.Errorf("execution %d not found with retention event", task.ExecutionID)
+	}
+
+	if execution.DryRun {
+		return nil, true, 0, nil
+	}
+
+	md, err := r.RetentionController().GetRetention(execution.PolicyID)
+	if err != nil {
+		log.Errorf("failed to get tag retention policy %d: error: %v", execution.PolicyID, err)
+		return nil, false, 0, err
+	}
+	if md == nil {
+		return nil, false, 0, fmt.Errorf("policy %d not found with tag retention event", execution.PolicyID)
+	}
+
+	extURL, err := config.ExtURL()
+	if err != nil {
+		log.Errorf("Error while reading external endpoint URL: %v", err)
+	}
+	hostname := strings.Split(extURL, ":")[0]
+
+	payload := &model.Payload{
+		Type:     event.EventType,
+		OccurAt:  event.OccurAt.Unix(),
+		Operator: execution.Trigger,
+		EventData: &model.EventData{
+			Retention: &evtModel.Retention{
+				Total:             task.Total,
+				Retained:          task.Retained,
+				HarborHostname:    hostname,
+				ProjectName:       event.Deleted[0].Target.Namespace,
+				RetentionPolicyID: execution.PolicyID,
+				Status:            event.Status,
+				RetentionRules:    []*evtModel.RetentionRule{},
+			},
+		},
+	}
+
+	for _, v := range event.Deleted {
+		target := v.Target
+		deletedArtifact := &evtModel.ArtifactInfo{
+			Type:   target.Kind,
+			Status: event.Status,
+		}
+		if len(target.Tags) != 0 {
+			deletedArtifact.NameAndTag = target.Repository + ":" + target.Tags[0]
+		}
+		payload.EventData.Retention.DeletedArtifact = []*evtModel.ArtifactInfo{deletedArtifact}
+	}
+
+	for _, v := range md.Rules {
+		retentionRule := &evtModel.RetentionRule{
+			Template:       v.Template,
+			Parameters:     v.Parameters,
+			TagSelectors:   v.TagSelectors,
+			ScopeSelectors: v.ScopeSelectors,
+		}
+		payload.EventData.Retention.RetentionRules = append(payload.EventData.Retention.RetentionRules, retentionRule)
+	}
+
+	return payload, false, event.Deleted[0].Target.NamespaceID, nil
+}

src/controller/event/handler/webhook/artifact/retention_test.go

@@ -0,0 +1,82 @@
+package artifact
+
+import (
+	"testing"
+	"time"
+
+	"github.com/goharbor/harbor/src/pkg/retention"
+
+	"github.com/goharbor/harbor/src/controller/event"
+	"github.com/goharbor/harbor/src/core/config"
+	"github.com/goharbor/harbor/src/lib/selector"
+	"github.com/goharbor/harbor/src/pkg/notification"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRetentionHandler_Handle(t *testing.T) {
+	config.Init()
+	handler := &RetentionHandler{RetentionController: DefaultRetentionControllerFunc}
+
+	policyMgr := notification.PolicyMgr
+	retentionCtlFunc := handler.RetentionController
+
+	defer func() {
+		notification.PolicyMgr = policyMgr
+		handler.RetentionController = retentionCtlFunc
+	}()
+	notification.PolicyMgr = &fakedNotificationPolicyMgr{}
+	handler.RetentionController = retention.FakedRetentionControllerFunc
+
+	type args struct {
+		data interface{}
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "RetentionHandler Want Error 1",
+			args: args{
+				data: "",
+			},
+			wantErr: true,
+		},
+		{
+			name: "RetentionHandler 1",
+			args: args{
+				data: &event.RetentionEvent{
+					OccurAt: time.Now(),
+					Deleted: []*selector.Result{
+						{
+							Target: &selector.Candidate{
+								NamespaceID: 1,
+								Namespace:   "project1",
+								Tags:        []string{"v1"},
+								Labels:      nil,
+							},
+						},
+					},
+				},
+			},
+			wantErr: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := handler.Handle(tt.args.data)
+			if tt.wantErr {
+				require.NotNil(t, err, "Error: %s", err)
+				return
+			}
+			assert.Nil(t, err)
+		})
+	}
+
+}
+
+func TestRetentionHandler_IsStateful(t *testing.T) {
+	handler := &RetentionHandler{}
+	assert.False(t, handler.IsStateful())
+}

src/controller/event/metadata/retention.go

@@ -0,0 +1,33 @@
+package metadata
+
+import (
+	"time"
+
+	event2 "github.com/goharbor/harbor/src/controller/event"
+	"github.com/goharbor/harbor/src/lib/selector"
+	"github.com/goharbor/harbor/src/pkg/notifier/event"
+)
+
+// RetentionMetaData defines tag retention related event data
+type RetentionMetaData struct {
+	Total    int
+	Retained int
+	Deleted  []*selector.Result
+	Status   string
+	TaskID   int64
+}
+
+// Resolve tag retention metadata into tag retention event
+func (r *RetentionMetaData) Resolve(evt *event.Event) error {
+	data := &event2.RetentionEvent{
+		EventType: event2.TopicTagRetention,
+		OccurAt:   time.Now(),
+		Status:    r.Status,
+		Deleted:   r.Deleted,
+		TaskID:    r.TaskID,
+	}
+
+	evt.Topic = event2.TopicTagRetention
+	evt.Data = data
+	return nil
+}

src/controller/event/metadata/retention_test.go

@@ -0,0 +1,34 @@
+package metadata
+
+import (
+	"testing"
+
+	event2 "github.com/goharbor/harbor/src/controller/event"
+	"github.com/goharbor/harbor/src/pkg/notifier/event"
+	"github.com/stretchr/testify/suite"
+)
+
+type retentionEventTestSuite struct {
+	suite.Suite
+}
+
+func (r *retentionEventTestSuite) TestResolveOfDeleteRepositoryEventMetadata() {
+	e := &event.Event{}
+	metadata := &RetentionMetaData{
+		Total:    0,
+		Retained: 0,
+		Deleted:  nil,
+		Status:   "",
+		TaskID:   0,
+	}
+	err := metadata.Resolve(e)
+	r.Require().Nil(err)
+	r.Equal(event2.TopicTagRetention, e.Topic)
+	r.Require().NotNil(e.Data)
+	_, ok := e.Data.(*event2.RetentionEvent)
+	r.Require().True(ok)
+}
+
+func TestRetentionEventTestSuite(t *testing.T) {
+	suite.Run(t, &retentionEventTestSuite{})
+}

src/controller/event/model/event.go

@@ -0,0 +1,61 @@
+package model
+
+import "github.com/goharbor/harbor/src/pkg/retention/policy/rule"
+
+// Replication describes replication infos
+type Replication struct {
+	HarborHostname     string               `json:"harbor_hostname,omitempty"`
+	JobStatus          string               `json:"job_status,omitempty"`
+	Description        string               `json:"description,omitempty"`
+	ArtifactType       string               `json:"artifact_type,omitempty"`
+	AuthenticationType string               `json:"authentication_type,omitempty"`
+	OverrideMode       bool                 `json:"override_mode,omitempty"`
+	TriggerType        string               `json:"trigger_type,omitempty"`
+	PolicyCreator      string               `json:"policy_creator,omitempty"`
+	ExecutionTimestamp int64                `json:"execution_timestamp,omitempty"`
+	SrcResource        *ReplicationResource `json:"src_resource,omitempty"`
+	DestResource       *ReplicationResource `json:"dest_resource,omitempty"`
+	SuccessfulArtifact []*ArtifactInfo      `json:"successful_artifact,omitempty"`
+	FailedArtifact     []*ArtifactInfo      `json:"failed_artifact,omitempty"`
+}
+
+// ArtifactInfo describe info of artifact
+type ArtifactInfo struct {
+	Type       string `json:"type"`
+	Status     string `json:"status"`
+	NameAndTag string `json:"name_tag"`
+	FailReason string `json:"fail_reason,omitempty"`
+}
+
+// ReplicationResource describes replication resource info
+type ReplicationResource struct {
+	RegistryName string `json:"registry_name,omitempty"`
+	RegistryType string `json:"registry_type"`
+	Endpoint     string `json:"endpoint"`
+	Provider     string `json:"provider,omitempty"`
+	Namespace    string `json:"namespace,omitempty"`
+}
+
+// Retention describes tag retention infos
+type Retention struct {
+	Total             int              `json:"total"`
+	Retained          int              `json:"retained"`
+	HarborHostname    string           `json:"harbor_hostname,omitempty"`
+	ProjectName       string           `json:"project_name,omitempty"`
+	RetentionPolicyID int64            `json:"retention_policy_id,omitempty"`
+	RetentionRules    []*RetentionRule `json:"retention_rule,omitempty"`
+	Status            string           `json:"result,omitempty"`
+	DeletedArtifact   []*ArtifactInfo  `json:"deleted_artifact,omitempty"`
+}
+
+// RetentionRule describes tag retention rule
+type RetentionRule struct {
+	// Template ID
+	Template string `json:"template,omitempty"`
+	// The parameters of this rule
+	Parameters map[string]rule.Parameter `json:"params,omitempty"`
+	// Selector attached to the rule for filtering tags
+	TagSelectors []*rule.Selector `json:"tag_selectors,omitempty" `
+	// Selector attached to the rule for filtering scope (e.g: repositories or namespaces)
+	ScopeSelectors map[string][]*rule.Selector `json:"scope_selectors,omitempty"`
+}

src/controller/event/topic.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/lib/selector"
 	"github.com/goharbor/harbor/src/pkg/artifact"
 	"github.com/goharbor/harbor/src/pkg/audit/model"
 	v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
@@ -46,6 +47,7 @@ const (
 	TopicDeleteChart     = "DELETE_CHART"
 	TopicReplication     = "REPLICATION"
 	TopicArtifactLabeled = "ARTIFACT_LABELED"
+	TopicTagRetention    = "TAG_RETENTION"
 )
 
 // CreateProjectEvent is the creating project event
@@ -289,3 +291,12 @@ type ArtifactLabeledEvent struct {
 	OccurAt    time.Time
 	Operator   string
 }
+
+// RetentionEvent is tag retention related event data to publish
+type RetentionEvent struct {
+	TaskID    int64
+	EventType string
+	OccurAt   time.Time
+	Status    string
+	Deleted   []*selector.Result
+}

src/controller/p2p/preheat/enforcer.go

@@ -476,7 +476,7 @@ func (de *defaultEnforcer) startTask(ctx context.Context, executionID int64, can
 
 // getVulnerabilitySev gets the severity code value for the given artifact with allowlist option set
 func (de *defaultEnforcer) getVulnerabilitySev(ctx context.Context, p *models.Project, art *artifact.Artifact) (uint, error) {
-	al := report.CVESet(p.CVEAllowlist.CVESet())
+	al := p.CVEAllowlist.CVESet()
 	r, err := de.scanCtl.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}, report.WithCVEAllowlist(&al))
 	if err != nil {
 		if errors.IsNotFoundErr(err) {

src/core/api/notification_policy.go

@@ -384,6 +384,7 @@ func initSupportedEvents() map[string]struct{} {
 		event.TopicScanningFailed,
 		event.TopicScanningCompleted,
 		event.TopicReplication,
+		event.TopicTagRetention,
 	}
 
 	var supportedEventTypes = make(map[string]struct{})

src/core/api/retention.go

@@ -4,10 +4,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/goharbor/harbor/src/core/config"
 	"net/http"
 	"strconv"
 
+	"github.com/goharbor/harbor/src/core/config"
+
 	"github.com/goharbor/harbor/src/common/rbac"
 	"github.com/goharbor/harbor/src/core/promgr"
 	"github.com/goharbor/harbor/src/pkg/retention"

src/core/service/notifications/jobs/handler.go

@@ -18,15 +18,15 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/goharbor/harbor/src/core/service/notifications"
-
 	"github.com/goharbor/harbor/src/common/job"
 	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/controller/event/metadata"
 	"github.com/goharbor/harbor/src/controller/scan"
+	"github.com/goharbor/harbor/src/core/service/notifications"
 	jjob "github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/lib/selector"
 	"github.com/goharbor/harbor/src/pkg/notification"
 	"github.com/goharbor/harbor/src/pkg/notifier/event"
 	"github.com/goharbor/harbor/src/pkg/retention"
@@ -188,8 +188,9 @@ func (h *Handler) HandleRetentionTask() {
 	// handle checkin
 	if h.checkIn != "" {
 		var retainObj struct {
-			Total    int `json:"total"`
-			Retained int `json:"retained"`
+			Total    int                `json:"total"`
+			Retained int                `json:"retained"`
+			Deleted  []*selector.Result `json:"deleted"`
 		}
 		if err := json.Unmarshal([]byte(h.checkIn), &retainObj); err != nil {
 			log.Errorf("failed to resolve checkin of retention task %d: %v", taskID, err)
@@ -205,6 +206,23 @@ func (h *Handler) HandleRetentionTask() {
 			h.SendInternalServerError(err)
 			return
 		}
+
+		e := &event.Event{}
+		metaData := &metadata.RetentionMetaData{
+			Total:    retainObj.Total,
+			Retained: retainObj.Retained,
+			Deleted:  retainObj.Deleted,
+			Status:   "SUCCESS",
+			TaskID:   taskID,
+		}
+
+		if err := e.Build(metaData); err == nil {
+			if err := e.Publish(); err != nil {
+				log.Error(errors.Wrap(err, "tag retention job hook handler: event publish"))
+			}
+		} else {
+			log.Error(errors.Wrap(err, "tag retention job hook handler: event publish"))
+		}
 		return
 	}
 

src/pkg/notifier/model/event.go

@@ -2,6 +2,7 @@ package model
 
 import (
 	"github.com/goharbor/harbor/src/common/models"
+	"github.com/goharbor/harbor/src/controller/event/model"
 )
 
 // HookEvent is hook related event data to publish
@@ -22,10 +23,11 @@ type Payload struct {
 
 // EventData of notification event payload
 type EventData struct {
-	Resources   []*Resource       `json:"resources,omitempty"`
-	Repository  *Repository       `json:"repository,omitempty"`
-	Replication *Replication      `json:"replication,omitempty"`
-	Custom      map[string]string `json:"custom_attributes,omitempty"`
+	Resources   []*Resource        `json:"resources,omitempty"`
+	Repository  *Repository        `json:"repository,omitempty"`
+	Replication *model.Replication `json:"replication,omitempty"`
+	Retention   *model.Retention   `json:"retention,omitempty"`
+	Custom      map[string]string  `json:"custom_attributes,omitempty"`
 }
 
 // Resource describe infos of resource triggered notification
@@ -44,37 +46,3 @@ type Repository struct {
 	RepoFullName string `json:"repo_full_name"`
 	RepoType     string `json:"repo_type"`
 }
-
-// Replication describes replication infos
-type Replication struct {
-	HarborHostname     string               `json:"harbor_hostname,omitempty"`
-	JobStatus          string               `json:"job_status,omitempty"`
-	Description        string               `json:"description,omitempty"`
-	ArtifactType       string               `json:"artifact_type,omitempty"`
-	AuthenticationType string               `json:"authentication_type,omitempty"`
-	OverrideMode       bool                 `json:"override_mode,omitempty"`
-	TriggerType        string               `json:"trigger_type,omitempty"`
-	PolicyCreator      string               `json:"policy_creator,omitempty"`
-	ExecutionTimestamp int64                `json:"execution_timestamp,omitempty"`
-	SrcResource        *ReplicationResource `json:"src_resource,omitempty"`
-	DestResource       *ReplicationResource `json:"dest_resource,omitempty"`
-	SuccessfulArtifact []*ArtifactInfo      `json:"successful_artifact,omitempty"`
-	FailedArtifact     []*ArtifactInfo      `json:"failed_artifact,omitempty"`
-}
-
-// ArtifactInfo describe info of artifact replicated
-type ArtifactInfo struct {
-	Type       string `json:"type"`
-	Status     string `json:"status"`
-	NameAndTag string `json:"name_tag"`
-	FailReason string `json:"fail_reason,omitempty"`
-}
-
-// ReplicationResource describes replication resource info
-type ReplicationResource struct {
-	RegistryName string `json:"registry_name,omitempty"`
-	RegistryType string `json:"registry_type"`
-	Endpoint     string `json:"endpoint"`
-	Provider     string `json:"provider,omitempty"`
-	Namespace    string `json:"namespace,omitempty"`
-}

src/pkg/retention/controller.go

@@ -53,6 +53,8 @@ type APIController interface {
 	GetTotalOfRetentionExecTasks(executionID int64) (int64, error)
 
 	GetRetentionExecTaskLog(taskID int64) ([]byte, error)
+
+	GetRetentionExecTask(taskID int64) (*Task, error)
 }
 
 // DefaultAPIController ...
@@ -255,6 +257,11 @@ func (r *DefaultAPIController) GetRetentionExecTaskLog(taskID int64) ([]byte, er
 	return r.manager.GetTaskLog(taskID)
 }
 
+// GetRetentionExecTask Get Retention Execution Task
+func (r *DefaultAPIController) GetRetentionExecTask(taskID int64) (*Task, error) {
+	return r.manager.GetTask(taskID)
+}
+
 // NewAPIController ...
 func NewAPIController(retentionMgr Manager, projectManager project.Manager, repositoryMgr repository.Manager, scheduler scheduler.Scheduler, retentionLauncher Launcher) APIController {
 	return &DefaultAPIController{

src/pkg/retention/faked_controller.go

@@ -0,0 +1,96 @@
+package retention
+
+import (
+	"github.com/goharbor/harbor/src/pkg/retention/policy"
+	"github.com/goharbor/harbor/src/pkg/retention/q"
+)
+
+// FakedRetentionController ...
+type FakedRetentionController struct {
+}
+
+// FakedRetentionControllerFunc ...
+var FakedRetentionControllerFunc = NewFakedRetentionController
+
+// NewFakedRetentionController ...
+func NewFakedRetentionController() APIController {
+	return &FakedRetentionController{}
+}
+
+// GetRetention ...
+func (f *FakedRetentionController) GetRetention(id int64) (*policy.Metadata, error) {
+	return &policy.Metadata{
+		ID:        1,
+		Algorithm: "",
+		Rules:     nil,
+		Trigger:   nil,
+		Scope:     nil,
+	}, nil
+}
+
+// CreateRetention ...
+func (f *FakedRetentionController) CreateRetention(p *policy.Metadata) (int64, error) {
+	return 0, nil
+}
+
+// UpdateRetention ...
+func (f *FakedRetentionController) UpdateRetention(p *policy.Metadata) error {
+	return nil
+}
+
+// DeleteRetention ...
+func (f *FakedRetentionController) DeleteRetention(id int64) error {
+	return nil
+}
+
+// TriggerRetentionExec ...
+func (f *FakedRetentionController) TriggerRetentionExec(policyID int64, trigger string, dryRun bool) (int64, error) {
+
+	return 0, nil
+}
+
+// OperateRetentionExec ...
+func (f *FakedRetentionController) OperateRetentionExec(eid int64, action string) error {
+	return nil
+}
+
+// GetRetentionExec ...
+func (f *FakedRetentionController) GetRetentionExec(eid int64) (*Execution, error) {
+	return &Execution{
+		DryRun:   false,
+		PolicyID: 1,
+	}, nil
+}
+
+// ListRetentionExecs ...
+func (f *FakedRetentionController) ListRetentionExecs(policyID int64, query *q.Query) ([]*Execution, error) {
+	return nil, nil
+}
+
+// GetTotalOfRetentionExecs ...
+func (f *FakedRetentionController) GetTotalOfRetentionExecs(policyID int64) (int64, error) {
+	return 0, nil
+}
+
+// ListRetentionExecTasks ...
+func (f *FakedRetentionController) ListRetentionExecTasks(executionID int64, query *q.Query) ([]*Task, error) {
+	return nil, nil
+}
+
+// GetTotalOfRetentionExecTasks ...
+func (f *FakedRetentionController) GetTotalOfRetentionExecTasks(executionID int64) (int64, error) {
+	return 0, nil
+}
+
+// GetRetentionExecTaskLog ...
+func (f *FakedRetentionController) GetRetentionExecTaskLog(taskID int64) ([]byte, error) {
+	return nil, nil
+}
+
+// GetRetentionExecTask ...
+func (f *FakedRetentionController) GetRetentionExecTask(taskID int64) (*Task, error) {
+	return &Task{
+		ID:          1,
+		ExecutionID: 1,
+	}, nil
+}

src/pkg/retention/mocks/api_controller.go

@@ -119,6 +119,14 @@ func (_m *APIController) GetRetentionExecTaskLog(taskID int64) ([]byte, error) {
 	return r0, r1
 }
 
+// GetRetentionExecTask provides a mock function with given fields: taskID
+func (_m *APIController) GetRetentionExecTask(taskID int64) (*retention.Task, error) {
+	return &retention.Task{
+		ID:          1,
+		ExecutionID: 1,
+	}, nil
+}
+
 // GetTotalOfRetentionExecTasks provides a mock function with given fields: executionID
 func (_m *APIController) GetTotalOfRetentionExecTasks(executionID int64) (int64, error) {
 	ret := _m.Called(executionID)

src/pkg/scan/report/summary.go

@@ -17,6 +17,7 @@ package report
 import (
 	"reflect"
 
+	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/jobservice/job"
 	"github.com/goharbor/harbor/src/lib/errors"
 	"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
@@ -24,29 +25,19 @@ import (
 	"github.com/goharbor/harbor/src/pkg/scan/vuln"
 )
 
-// CVESet defines the CVE allowlist with a hash set way for easy query.
-type CVESet map[string]struct{}
-
-// Contains checks whether the specified CVE is in the set or not.
-func (cs CVESet) Contains(cve string) bool {
-	_, ok := cs[cve]
-
-	return ok
-}
-
 // Options provides options for getting the report w/ summary.
 type Options struct {
 	// If it is set, the returned report will contains artifact digest for the vulnerabilities
 	ArtifactDigest string
 	// If it is set, the returned summary will not count the CVEs in the list in.
-	CVEAllowlist CVESet
+	CVEAllowlist models.CVESet
 }
 
 // Option for getting the report w/ summary with func template way.
 type Option func(options *Options)
 
 // WithCVEAllowlist is an option of setting CVE allowlist.
-func WithCVEAllowlist(set *CVESet) Option {
+func WithCVEAllowlist(set *models.CVESet) Option {
 	return func(options *Options) {
 		options.CVEAllowlist = *set
 	}

src/pkg/scan/report/summary_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/goharbor/harbor/src/common/models"
 	"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
 	v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
 	"github.com/goharbor/harbor/src/pkg/scan/vuln"
@@ -108,7 +109,7 @@ func (suite *SummaryTestSuite) TestSummaryGenerateSummaryNoOptions() {
 
 // TestSummaryGenerateSummaryWithOptions ...
 func (suite *SummaryTestSuite) TestSummaryGenerateSummaryWithOptions() {
-	cveSet := make(CVESet)
+	cveSet := make(models.CVESet)
 	cveSet["2019-0980-0909"] = struct{}{}
 
 	summaries, err := GenerateSummary(suite.r, WithCVEAllowlist(&cveSet))

src/portal/src/app/project/webhook/webhook.service.ts

@@ -30,6 +30,7 @@ const EVENT_TYPES_TEXT_MAP = {
   'QUOTA_WARNING': 'Quota near threshold',
   'SCANNING_FAILED': 'Scanning failed',
   'SCANNING_COMPLETED': 'Scanning finished',
+  'TAG_RETENTION': 'Tag retention finished',
 };
 
 @Injectable()

src/server/middleware/repoproxy/proxy.go

@@ -184,7 +184,7 @@ func DisableBlobAndManifestUploadMiddleware() func(http.Handler) http.Handler {
 		}
 		if isProxyProject(p) && !isProxySession(ctx) {
 			httpLib.SendError(w,
-				errors.MethodNotAllowedError(
+				errors.DeniedError(
 					errors.Errorf("can not push artifact to a proxy project: %v", p.Name)))
 			return
 		}

src/server/middleware/vulnerable/vulnerable.go

@@ -91,7 +91,7 @@ func Middleware() func(http.Handler) http.Handler {
 			return nil
 		}
 
-		allowlist := report.CVESet(proj.CVEAllowlist.CVESet())
+		allowlist := proj.CVEAllowlist.CVESet()
 		summaries, err := scanController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}, report.WithCVEAllowlist(&allowlist))
 		if err != nil {
 			logger.Errorf("get vulnerability summary of the artifact %s@%s failed, error: %v", art.RepositoryName, art.Digest, err)

src/server/registry/route.go

@@ -77,6 +77,7 @@ func RegisterRoutes() {
 	root.NewRoute().
 		Method(http.MethodPost).
 		Path("/*/blobs/uploads").
+		Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()).
 		Middleware(quota.PostInitiateBlobUploadMiddleware()).
 		Middleware(blob.PostInitiateBlobUploadMiddleware()).
 		Handler(proxy)
@@ -84,13 +85,11 @@ func RegisterRoutes() {
 	root.NewRoute().
 		Method(http.MethodPatch).
 		Path("/*/blobs/uploads/:session_id").
-		Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()).
 		Middleware(blob.PatchBlobUploadMiddleware()).
 		Handler(proxy)
 	root.NewRoute().
 		Method(http.MethodPut).
 		Path("/*/blobs/uploads/:session_id").
-		Middleware(repoproxy.DisableBlobAndManifestUploadMiddleware()).
 		Middleware(quota.PutBlobUploadMiddleware()).
 		Middleware(blob.PutBlobUploadMiddleware()).
 		Handler(proxy)

src/server/v2.0/handler/artifact.go

@@ -155,14 +155,9 @@ func (a *artifactAPI) CopyArtifact(ctx context.Context, params operation.CopyArt
 		return a.SendError(ctx, err)
 	}
 
-	pro, err := a.proCtl.GetByName(ctx, params.ProjectName)
-	if err != nil {
+	if err := a.requireNonProxyCacheProject(ctx, params.ProjectName); err != nil {
 		return a.SendError(ctx, err)
 	}
-	if pro.RegistryID > 0 {
-		return a.SendError(ctx, errors.New(nil).WithCode(errors.MethodNotAllowedCode).
-			WithMessage("cannot copy the artifact to a proxy cache project"))
-	}
 
 	srcRepo, ref, err := parse(params.From)
 	if err != nil {
@@ -212,6 +207,11 @@ func (a *artifactAPI) CreateTag(ctx context.Context, params operation.CreateTagP
 	if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionCreate, rbac.ResourceTag); err != nil {
 		return a.SendError(ctx, err)
 	}
+
+	if err := a.requireNonProxyCacheProject(ctx, params.ProjectName); err != nil {
+		return a.SendError(ctx, err)
+	}
+
 	art, err := a.artCtl.GetByReference(ctx, fmt.Sprintf("%s/%s", params.ProjectName, params.RepositoryName),
 		params.Reference, &artifact.Option{
 			WithTag: true,
@@ -239,6 +239,18 @@ func (a *artifactAPI) CreateTag(ctx context.Context, params operation.CreateTagP
 	return operation.NewCreateTagCreated()
 }
 
+func (a *artifactAPI) requireNonProxyCacheProject(ctx context.Context, name string) error {
+	pro, err := a.proCtl.GetByName(ctx, name)
+	if err != nil {
+		return err
+	}
+	if pro.RegistryID > 0 {
+		return errors.New(nil).WithCode(errors.MethodNotAllowedCode).
+			WithMessage("the operation isn't supported for a proxy cache project")
+	}
+	return nil
+}
+
 func (a *artifactAPI) DeleteTag(ctx context.Context, params operation.DeleteTagParams) middleware.Responder {
 	if err := a.RequireProjectAccess(ctx, params.ProjectName, rbac.ActionDelete, rbac.ResourceTag); err != nil {
 		return a.SendError(ctx, err)

tests/ci/api_common_install.sh

@@ -30,7 +30,7 @@ if [ $GITHUB_TOKEN ];
 then
     sed "s/# github_token: xxx/github_token: $GITHUB_TOKEN/" -i make/harbor.yml
 fi
-sudo make compile build prepare COMPILETAG=compile_golangimage GOBUILDTAGS="include_oss include_gcs" NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true GEN_TLS=true
+sudo make build_base_docker compile build prepare COMPILETAG=compile_golangimage GOBUILDTAGS="include_oss include_gcs" NOTARYFLAG=true CLAIRFLAG=true TRIVYFLAG=true CHARTFLAG=true GEN_TLS=true
 
 # set the debugging env
 echo "GC_TIME_WINDOW_HOURS=0" | sudo tee -a ./make/common/config/core/env