diff --git a/src/pkg/art/candidate.go b/src/pkg/art/candidate.go index f44e22b99..e36950f17 100644 --- a/src/pkg/art/candidate.go +++ b/src/pkg/art/candidate.go @@ -32,6 +32,8 @@ const ( // Repository of candidate type Repository struct { + // Namespace(project) ID + NamespaceID int64 // Namespace Namespace string `json:"namespace"` // Repository name @@ -94,3 +96,10 @@ func (c *Candidate) Hash() string { return base64.StdEncoding.EncodeToString([]byte(raw)) } + +// NameHash based on the candidate info for differentiation +func (c *Candidate) NameHash() string { + raw := fmt.Sprintf("%s:%s/%s:%s", c.Kind, c.Namespace, c.Repository, c.Tag) + + return base64.StdEncoding.EncodeToString([]byte(raw)) +} diff --git a/src/pkg/art/result.go b/src/pkg/art/result.go index 43d09b29d..bd3d852a5 100644 --- a/src/pkg/art/result.go +++ b/src/pkg/art/result.go @@ -20,3 +20,15 @@ type Result struct { // nil error means success Error error `json:"error"` } + +// ImmutableError ... +type ImmutableError struct { + IsShareDigest bool +} + +func (e *ImmutableError) Error() string { + if e.IsShareDigest { + return "Same digest with other immutable tag" + } + return "Immutable tag" +} diff --git a/src/pkg/retention/dep/client.go b/src/pkg/retention/dep/client.go index 871b8a924..a7fc935e1 100644 --- a/src/pkg/retention/dep/client.go +++ b/src/pkg/retention/dep/client.go @@ -106,6 +106,7 @@ func (bc *basicClient) GetCandidates(repository *art.Repository) ([]*art.Candida } candidate := &art.Candidate{ Kind: art.Image, + NamespaceID: repository.NamespaceID, Namespace: repository.Namespace, Repository: repository.Name, Tag: image.Name, diff --git a/src/pkg/retention/job.go b/src/pkg/retention/job.go index 0944aa0c1..3e55f830a 100644 --- a/src/pkg/retention/job.go +++ b/src/pkg/retention/job.go @@ -32,9 +32,10 @@ import ( ) const ( - actionMarkRetain = "RETAIN" - actionMarkDeletion = "DEL" - actionMarkError = "ERR" + actionMarkRetain = "RETAIN" + actionMarkDeletion = "DEL" + actionMarkError = "ERR" + actionMarkImmutable = "IMMUTABLE" ) // Job of running retention process @@ -116,9 +117,9 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error { return saveRetainNum(ctx, results, allCandidates) } -func saveRetainNum(ctx job.Context, retained []*art.Result, allCandidates []*art.Candidate) error { +func saveRetainNum(ctx job.Context, results []*art.Result, allCandidates []*art.Candidate) error { var delNum int - for _, r := range retained { + for _, r := range results { if r.Error == nil { delNum++ } @@ -146,9 +147,12 @@ func logResults(logger logger.Interface, all []*art.Candidate, results []*art.Re } } - op := func(art *art.Candidate) string { - if e, exists := hash[art.Hash()]; exists { + op := func(c *art.Candidate) string { + if e, exists := hash[c.Hash()]; exists { if e != nil { + if _, ok := e.(*art.ImmutableError); ok { + return actionMarkImmutable + } return actionMarkError } diff --git a/src/pkg/retention/launcher.go b/src/pkg/retention/launcher.go index 09d264ee7..30a5e4ba6 100644 --- a/src/pkg/retention/launcher.go +++ b/src/pkg/retention/launcher.go @@ -175,9 +175,10 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool for _, repositoryCandidate := range repositoryCandidates { reposit := art.Repository{ - Namespace: repositoryCandidate.Namespace, - Name: repositoryCandidate.Repository, - Kind: repositoryCandidate.Kind, + NamespaceID: repositoryCandidate.NamespaceID, + Namespace: repositoryCandidate.Namespace, + Name: repositoryCandidate.Repository, + Kind: repositoryCandidate.Kind, } if repositoryRules[reposit] == nil { repositoryRules[reposit] = &lwp.Metadata{ @@ -352,9 +353,10 @@ func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manage for _, r := range imageRepositories { namespace, repo := utils.ParseRepository(r.Name) candidates = append(candidates, &art.Candidate{ - Namespace: namespace, - Repository: repo, - Kind: "image", + NamespaceID: projectID, + Namespace: namespace, + Repository: repo, + Kind: "image", }) } // currently, doesn't support retention for chart diff --git a/src/pkg/retention/policy/action/performer.go b/src/pkg/retention/policy/action/performer.go index 2461d0945..2c5ba4258 100644 --- a/src/pkg/retention/policy/action/performer.go +++ b/src/pkg/retention/policy/action/performer.go @@ -15,7 +15,10 @@ package action import ( + "github.com/goharbor/harbor/src/common/utils" + "github.com/goharbor/harbor/src/common/utils/log" "github.com/goharbor/harbor/src/pkg/art" + "github.com/goharbor/harbor/src/pkg/immutabletag/match/rule" "github.com/goharbor/harbor/src/pkg/retention/dep" ) @@ -50,25 +53,45 @@ type retainAction struct { // Perform the action func (ra *retainAction) Perform(candidates []*art.Candidate) (results []*art.Result, err error) { retained := make(map[string]bool) + immutable := make(map[string]bool) + retainedShare := make(map[string]bool) + immutableShare := make(map[string]bool) for _, c := range candidates { - retained[c.Hash()] = true + retained[c.NameHash()] = true + retainedShare[c.Hash()] = true + } + + for _, c := range ra.all { + if _, ok := retainedShare[c.Hash()]; ok { + continue + } + if isImmutable(c) { + immutable[c.NameHash()] = true + immutableShare[c.Hash()] = true + } } // start to delete if len(ra.all) > 0 { for _, c := range ra.all { - if _, ok := retained[c.Hash()]; !ok { - result := &art.Result{ - Target: c, - } - - if !ra.isDryRun { - if err := dep.DefaultClient.Delete(c); err != nil { - result.Error = err + if _, ok := retained[c.NameHash()]; !ok { + if _, ok = retainedShare[c.Hash()]; !ok { + result := &art.Result{ + Target: c, } + if _, ok = immutable[c.NameHash()]; ok { + result.Error = &art.ImmutableError{} + } else if _, ok = immutableShare[c.Hash()]; ok { + result.Error = &art.ImmutableError{IsShareDigest: true} + } else { + if !ra.isDryRun { + if err := dep.DefaultClient.Delete(c); err != nil { + result.Error = err + } + } + } + results = append(results, result) } - - results = append(results, result) } } } @@ -76,6 +99,23 @@ func (ra *retainAction) Perform(candidates []*art.Candidate) (results []*art.Res return } +func isImmutable(c *art.Candidate) bool { + projectID := c.NamespaceID + repo := c.Repository + tag := c.Tag + _, repoName := utils.ParseRepository(repo) + matched, err := rule.NewRuleMatcher(projectID).Match(art.Candidate{ + Repository: repoName, + Tag: tag, + NamespaceID: projectID, + }) + if err != nil { + log.Error(err) + return false + } + return matched +} + // NewRetainAction is factory method for RetainAction func NewRetainAction(params interface{}, isDryRun bool) Performer { if params != nil { diff --git a/src/pkg/retention/policy/action/performer_test.go b/src/pkg/retention/policy/action/performer_test.go index 868bb4c93..8bfef863e 100644 --- a/src/pkg/retention/policy/action/performer_test.go +++ b/src/pkg/retention/policy/action/performer_test.go @@ -15,10 +15,13 @@ package action import ( + "github.com/goharbor/harbor/src/common/dao" + "github.com/goharbor/harbor/src/pkg/immutabletag" "testing" "time" "github.com/goharbor/harbor/src/pkg/art" + immumodel "github.com/goharbor/harbor/src/pkg/immutabletag/model" "github.com/goharbor/harbor/src/pkg/retention/dep" "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -64,6 +67,7 @@ func (suite *TestPerformerSuite) SetupSuite() { suite.oldClient = dep.DefaultClient dep.DefaultClient = &fakeRetentionClient{} + dao.PrepareTestForPostgresSQL() } // TearDownSuite ... @@ -97,6 +101,122 @@ func (suite *TestPerformerSuite) TestPerform() { assert.Equal(suite.T(), "dev", results[0].Target.Tag) } +// TestPerform tests Perform action +func (suite *TestPerformerSuite) TestPerformImmutable() { + all := []*art.Candidate{ + { + NamespaceID: 1, + Namespace: "library", + Repository: "harbor", + Kind: "image", + Tag: "latest", + Digest: "d0", + PushedTime: time.Now().Unix(), + Labels: []string{"L1", "L2"}, + }, + { + NamespaceID: 1, + Namespace: "library", + Repository: "harbor", + Kind: "image", + Tag: "dev", + Digest: "d1", + PushedTime: time.Now().Unix(), + Labels: []string{"L3"}, + }, + { + NamespaceID: 1, + Namespace: "library", + Repository: "test", + Kind: "image", + Tag: "immute", + Digest: "d2", + PushedTime: time.Now().Unix(), + Labels: []string{"L1", "L2"}, + }, + { + NamespaceID: 1, + Namespace: "library", + Repository: "test", + Kind: "image", + Tag: "samedig", + Digest: "d2", + PushedTime: time.Now().Unix(), + Labels: []string{"L1", "L2"}, + }, + } + p := &retainAction{ + all: all, + } + + rule := &immumodel.Metadata{ + ProjectID: 1, + Priority: 1, + Action: "immutable", + Template: "immutable_template", + TagSelectors: []*immumodel.Selector{ + { + Kind: "doublestar", + Decoration: "matches", + Pattern: "immute", + }, + }, + ScopeSelectors: map[string][]*immumodel.Selector{ + "repository": { + { + Kind: "doublestar", + Decoration: "repoMatches", + Pattern: "**", + }, + }, + }, + } + imid, e := immutabletag.ImmuCtr.CreateImmutableRule(rule) + assert.NoError(suite.T(), e) + defer func() { + assert.NoError(suite.T(), immutabletag.ImmuCtr.DeleteImmutableRule(imid)) + }() + + candidates := []*art.Candidate{ + { + NamespaceID: 1, + Namespace: "library", + Repository: "harbor", + Kind: "image", + Tag: "latest", + Digest: "d0", + PushedTime: time.Now().Unix(), + Labels: []string{"L1", "L2"}, + }, + } + + results, err := p.Perform(candidates) + require.NoError(suite.T(), err) + require.Equal(suite.T(), 3, len(results)) + for _, r := range results { + require.NotNil(suite.T(), r.Target) + if r.Target.Digest == "d1" { + require.NoError(suite.T(), r.Error) + require.Equal(suite.T(), "dev", r.Target.Tag) + } else if r.Target.Digest == "d2" { + require.Error(suite.T(), r.Error) + require.IsType(suite.T(), (*art.ImmutableError)(nil), r.Error) + if i, ok := r.Error.(*art.ImmutableError); ok { + if r.Target.Tag == "immute" { + require.False(suite.T(), i.IsShareDigest) + } else { + require.True(suite.T(), i.IsShareDigest) + } + } + } else { + require.Fail(suite.T(), "should not delete "+r.Target.NameHash()) + } + } + require.NotNil(suite.T(), results[0].Target) + assert.NoError(suite.T(), results[0].Error) + assert.Equal(suite.T(), "dev", results[0].Target.Tag) +} + type fakeRetentionClient struct{} // GetCandidates ... diff --git a/src/pkg/retention/policy/alg/or/processor_test.go b/src/pkg/retention/policy/alg/or/processor_test.go index 54e5233f5..111f80a3a 100644 --- a/src/pkg/retention/policy/alg/or/processor_test.go +++ b/src/pkg/retention/policy/alg/or/processor_test.go @@ -16,6 +16,7 @@ package or import ( "errors" + "github.com/goharbor/harbor/src/common/dao" "testing" "time" @@ -50,6 +51,7 @@ func TestProcessor(t *testing.T) { // SetupSuite ... func (suite *ProcessorTestSuite) SetupSuite() { + dao.PrepareTestForPostgresSQL() suite.all = []*art.Candidate{ { Namespace: "library", diff --git a/src/pkg/retention/policy/builder_test.go b/src/pkg/retention/policy/builder_test.go index 60ba74e0e..b55f2c8a7 100644 --- a/src/pkg/retention/policy/builder_test.go +++ b/src/pkg/retention/policy/builder_test.go @@ -15,6 +15,7 @@ package policy import ( + "github.com/goharbor/harbor/src/common/dao" "testing" "time" @@ -66,6 +67,7 @@ func TestBuilder(t *testing.T) { // SetupSuite prepares the testing content if needed func (suite *TestBuilderSuite) SetupSuite() { + dao.PrepareTestForPostgresSQL() suite.all = []*art.Candidate{ { NamespaceID: 1, diff --git a/src/pkg/retention/policy/lwp/models.go b/src/pkg/retention/policy/lwp/models.go index 61d48efe5..6fe9044ed 100644 --- a/src/pkg/retention/policy/lwp/models.go +++ b/src/pkg/retention/policy/lwp/models.go @@ -25,6 +25,8 @@ import ( // Metadata contains partial metadata of policy // It's a lightweight version of policy.Metadata type Metadata struct { + // ID of the policy + ID int64 `json:"id"` // Algorithm applied to the rules // "OR" / "AND" Algorithm string `json:"algorithm"`