Merge pull request #9491 from bitsf/tag_retention_webhook

implement log for tag retention immutable tags

src/pkg/art/candidate.go

@@ -32,6 +32,8 @@ const (
 
 // Repository of candidate
 type Repository struct {
+	// Namespace(project) ID
+	NamespaceID int64
 	// Namespace
 	Namespace string `json:"namespace"`
 	// Repository name
@@ -94,3 +96,10 @@ func (c *Candidate) Hash() string {
 
 	return base64.StdEncoding.EncodeToString([]byte(raw))
 }
+
+// NameHash based on the candidate info for differentiation
+func (c *Candidate) NameHash() string {
+	raw := fmt.Sprintf("%s:%s/%s:%s", c.Kind, c.Namespace, c.Repository, c.Tag)
+
+	return base64.StdEncoding.EncodeToString([]byte(raw))
+}

src/pkg/art/result.go

@@ -20,3 +20,15 @@ type Result struct {
 	// nil error means success
 	Error error `json:"error"`
 }
+
+// ImmutableError ...
+type ImmutableError struct {
+	IsShareDigest bool
+}
+
+func (e *ImmutableError) Error() string {
+	if e.IsShareDigest {
+		return "Same digest with other immutable tag"
+	}
+	return "Immutable tag"
+}

src/pkg/retention/dep/client.go

@@ -106,6 +106,7 @@ func (bc *basicClient) GetCandidates(repository *art.Repository) ([]*art.Candida
 			}
 			candidate := &art.Candidate{
 				Kind:         art.Image,
+				NamespaceID:  repository.NamespaceID,
 				Namespace:    repository.Namespace,
 				Repository:   repository.Name,
 				Tag:          image.Name,

src/pkg/retention/job.go

@@ -32,9 +32,10 @@ import (
 )
 
 const (
-	actionMarkRetain   = "RETAIN"
-	actionMarkDeletion = "DEL"
-	actionMarkError    = "ERR"
+	actionMarkRetain    = "RETAIN"
+	actionMarkDeletion  = "DEL"
+	actionMarkError     = "ERR"
+	actionMarkImmutable = "IMMUTABLE"
 )
 
 // Job of running retention process
@@ -116,9 +117,9 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error {
 	return saveRetainNum(ctx, results, allCandidates)
 }
 
-func saveRetainNum(ctx job.Context, retained []*art.Result, allCandidates []*art.Candidate) error {
+func saveRetainNum(ctx job.Context, results []*art.Result, allCandidates []*art.Candidate) error {
 	var delNum int
-	for _, r := range retained {
+	for _, r := range results {
 		if r.Error == nil {
 			delNum++
 		}
@@ -146,9 +147,12 @@ func logResults(logger logger.Interface, all []*art.Candidate, results []*art.Re
 		}
 	}
 
-	op := func(art *art.Candidate) string {
-		if e, exists := hash[art.Hash()]; exists {
+	op := func(c *art.Candidate) string {
+		if e, exists := hash[c.Hash()]; exists {
 			if e != nil {
+				if _, ok := e.(*art.ImmutableError); ok {
+					return actionMarkImmutable
+				}
 				return actionMarkError
 			}
 

src/pkg/retention/launcher.go

@@ -175,9 +175,10 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool
 
 		for _, repositoryCandidate := range repositoryCandidates {
 			reposit := art.Repository{
-				Namespace: repositoryCandidate.Namespace,
-				Name:      repositoryCandidate.Repository,
-				Kind:      repositoryCandidate.Kind,
+				NamespaceID: repositoryCandidate.NamespaceID,
+				Namespace:   repositoryCandidate.Namespace,
+				Name:        repositoryCandidate.Repository,
+				Kind:        repositoryCandidate.Kind,
 			}
 			if repositoryRules[reposit] == nil {
 				repositoryRules[reposit] = &lwp.Metadata{
@@ -352,9 +353,10 @@ func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manage
 	for _, r := range imageRepositories {
 		namespace, repo := utils.ParseRepository(r.Name)
 		candidates = append(candidates, &art.Candidate{
-			Namespace:  namespace,
-			Repository: repo,
-			Kind:       "image",
+			NamespaceID: projectID,
+			Namespace:   namespace,
+			Repository:  repo,
+			Kind:        "image",
 		})
 	}
 	// currently, doesn't support retention for chart

src/pkg/retention/policy/action/performer.go

@@ -15,7 +15,10 @@
 package action
 
 import (
+	"github.com/goharbor/harbor/src/common/utils"
+	"github.com/goharbor/harbor/src/common/utils/log"
 	"github.com/goharbor/harbor/src/pkg/art"
+	"github.com/goharbor/harbor/src/pkg/immutabletag/match/rule"
 	"github.com/goharbor/harbor/src/pkg/retention/dep"
 )
 
@@ -50,25 +53,45 @@ type retainAction struct {
 // Perform the action
 func (ra *retainAction) Perform(candidates []*art.Candidate) (results []*art.Result, err error) {
 	retained := make(map[string]bool)
+	immutable := make(map[string]bool)
+	retainedShare := make(map[string]bool)
+	immutableShare := make(map[string]bool)
 	for _, c := range candidates {
-		retained[c.Hash()] = true
+		retained[c.NameHash()] = true
+		retainedShare[c.Hash()] = true
+	}
+
+	for _, c := range ra.all {
+		if _, ok := retainedShare[c.Hash()]; ok {
+			continue
+		}
+		if isImmutable(c) {
+			immutable[c.NameHash()] = true
+			immutableShare[c.Hash()] = true
+		}
 	}
 
 	// start to delete
 	if len(ra.all) > 0 {
 		for _, c := range ra.all {
-			if _, ok := retained[c.Hash()]; !ok {
-				result := &art.Result{
-					Target: c,
-				}
-
-				if !ra.isDryRun {
-					if err := dep.DefaultClient.Delete(c); err != nil {
-						result.Error = err
+			if _, ok := retained[c.NameHash()]; !ok {
+				if _, ok = retainedShare[c.Hash()]; !ok {
+					result := &art.Result{
+						Target: c,
 					}
+					if _, ok = immutable[c.NameHash()]; ok {
+						result.Error = &art.ImmutableError{}
+					} else if _, ok = immutableShare[c.Hash()]; ok {
+						result.Error = &art.ImmutableError{IsShareDigest: true}
+					} else {
+						if !ra.isDryRun {
+							if err := dep.DefaultClient.Delete(c); err != nil {
+								result.Error = err
+							}
+						}
+					}
+					results = append(results, result)
 				}
-
-				results = append(results, result)
 			}
 		}
 	}
@@ -76,6 +99,23 @@ func (ra *retainAction) Perform(candidates []*art.Candidate) (results []*art.Res
 	return
 }
 
+func isImmutable(c *art.Candidate) bool {
+	projectID := c.NamespaceID
+	repo := c.Repository
+	tag := c.Tag
+	_, repoName := utils.ParseRepository(repo)
+	matched, err := rule.NewRuleMatcher(projectID).Match(art.Candidate{
+		Repository:  repoName,
+		Tag:         tag,
+		NamespaceID: projectID,
+	})
+	if err != nil {
+		log.Error(err)
+		return false
+	}
+	return matched
+}
+
 // NewRetainAction is factory method for RetainAction
 func NewRetainAction(params interface{}, isDryRun bool) Performer {
 	if params != nil {

src/pkg/retention/policy/action/performer_test.go

@@ -15,10 +15,13 @@
 package action
 
 import (
+	"github.com/goharbor/harbor/src/common/dao"
+	"github.com/goharbor/harbor/src/pkg/immutabletag"
 	"testing"
 	"time"
 
 	"github.com/goharbor/harbor/src/pkg/art"
+	immumodel "github.com/goharbor/harbor/src/pkg/immutabletag/model"
 	"github.com/goharbor/harbor/src/pkg/retention/dep"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
@@ -64,6 +67,7 @@ func (suite *TestPerformerSuite) SetupSuite() {
 
 	suite.oldClient = dep.DefaultClient
 	dep.DefaultClient = &fakeRetentionClient{}
+	dao.PrepareTestForPostgresSQL()
 }
 
 // TearDownSuite ...
@@ -97,6 +101,122 @@ func (suite *TestPerformerSuite) TestPerform() {
 	assert.Equal(suite.T(), "dev", results[0].Target.Tag)
 }
 
+// TestPerform tests Perform action
+func (suite *TestPerformerSuite) TestPerformImmutable() {
+	all := []*art.Candidate{
+		{
+			NamespaceID: 1,
+			Namespace:   "library",
+			Repository:  "harbor",
+			Kind:        "image",
+			Tag:         "latest",
+			Digest:      "d0",
+			PushedTime:  time.Now().Unix(),
+			Labels:      []string{"L1", "L2"},
+		},
+		{
+			NamespaceID: 1,
+			Namespace:   "library",
+			Repository:  "harbor",
+			Kind:        "image",
+			Tag:         "dev",
+			Digest:      "d1",
+			PushedTime:  time.Now().Unix(),
+			Labels:      []string{"L3"},
+		},
+		{
+			NamespaceID: 1,
+			Namespace:   "library",
+			Repository:  "test",
+			Kind:        "image",
+			Tag:         "immute",
+			Digest:      "d2",
+			PushedTime:  time.Now().Unix(),
+			Labels:      []string{"L1", "L2"},
+		},
+		{
+			NamespaceID: 1,
+			Namespace:   "library",
+			Repository:  "test",
+			Kind:        "image",
+			Tag:         "samedig",
+			Digest:      "d2",
+			PushedTime:  time.Now().Unix(),
+			Labels:      []string{"L1", "L2"},
+		},
+	}
+	p := &retainAction{
+		all: all,
+	}
+
+	rule := &immumodel.Metadata{
+		ProjectID: 1,
+		Priority:  1,
+		Action:    "immutable",
+		Template:  "immutable_template",
+		TagSelectors: []*immumodel.Selector{
+			{
+				Kind:       "doublestar",
+				Decoration: "matches",
+				Pattern:    "immute",
+			},
+		},
+		ScopeSelectors: map[string][]*immumodel.Selector{
+			"repository": {
+				{
+					Kind:       "doublestar",
+					Decoration: "repoMatches",
+					Pattern:    "**",
+				},
+			},
+		},
+	}
+	imid, e := immutabletag.ImmuCtr.CreateImmutableRule(rule)
+	assert.NoError(suite.T(), e)
+	defer func() {
+		assert.NoError(suite.T(), immutabletag.ImmuCtr.DeleteImmutableRule(imid))
+	}()
+
+	candidates := []*art.Candidate{
+		{
+			NamespaceID: 1,
+			Namespace:   "library",
+			Repository:  "harbor",
+			Kind:        "image",
+			Tag:         "latest",
+			Digest:      "d0",
+			PushedTime:  time.Now().Unix(),
+			Labels:      []string{"L1", "L2"},
+		},
+	}
+
+	results, err := p.Perform(candidates)
+	require.NoError(suite.T(), err)
+	require.Equal(suite.T(), 3, len(results))
+	for _, r := range results {
+		require.NotNil(suite.T(), r.Target)
+		if r.Target.Digest == "d1" {
+			require.NoError(suite.T(), r.Error)
+			require.Equal(suite.T(), "dev", r.Target.Tag)
+		} else if r.Target.Digest == "d2" {
+			require.Error(suite.T(), r.Error)
+			require.IsType(suite.T(), (*art.ImmutableError)(nil), r.Error)
+			if i, ok := r.Error.(*art.ImmutableError); ok {
+				if r.Target.Tag == "immute" {
+					require.False(suite.T(), i.IsShareDigest)
+				} else {
+					require.True(suite.T(), i.IsShareDigest)
+				}
+			}
+		} else {
+			require.Fail(suite.T(), "should not delete "+r.Target.NameHash())
+		}
+	}
+	require.NotNil(suite.T(), results[0].Target)
+	assert.NoError(suite.T(), results[0].Error)
+	assert.Equal(suite.T(), "dev", results[0].Target.Tag)
+}
+
 type fakeRetentionClient struct{}
 
 // GetCandidates ...

src/pkg/retention/policy/alg/or/processor_test.go

@@ -16,6 +16,7 @@ package or
 
 import (
 	"errors"
+	"github.com/goharbor/harbor/src/common/dao"
 	"testing"
 	"time"
 
@@ -50,6 +51,7 @@ func TestProcessor(t *testing.T) {
 
 // SetupSuite ...
 func (suite *ProcessorTestSuite) SetupSuite() {
+	dao.PrepareTestForPostgresSQL()
 	suite.all = []*art.Candidate{
 		{
 			Namespace:  "library",

src/pkg/retention/policy/builder_test.go

@@ -15,6 +15,7 @@
 package policy
 
 import (
+	"github.com/goharbor/harbor/src/common/dao"
 	"testing"
 	"time"
 
@@ -66,6 +67,7 @@ func TestBuilder(t *testing.T) {
 
 // SetupSuite prepares the testing content if needed
 func (suite *TestBuilderSuite) SetupSuite() {
+	dao.PrepareTestForPostgresSQL()
 	suite.all = []*art.Candidate{
 		{
 			NamespaceID: 1,

src/pkg/retention/policy/lwp/models.go

@@ -25,6 +25,8 @@ import (
 // Metadata contains partial metadata of policy
 // It's a lightweight version of policy.Metadata
 type Metadata struct {
+	// ID of the policy
+	ID int64 `json:"id"`
 	// Algorithm applied to the rules
 	// "OR" / "AND"
 	Algorithm string `json:"algorithm"`