Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-12-18 22:57:38 +01:00
Code Issues Projects Releases Wiki Activity

add special error/log for not delete immutable tag in tag retention job

Browse Source 
Change-Id: I3440f3b888bf8c65afc75d04253eea41f20eef0e
Signed-off-by: Ziming Zhang <zziming@vmware.com>
This commit is contained in:
Ziming Zhang 2019-10-22 15:16:29 +08:00
parent c3c8b03af5
commit e757899b49
10 changed files with 218 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/pkg/art/candidate.go
View File

@ -32,6 +32,8 @@ const (
// Repository of candidate
type Repository struct {
// Namespace(project) ID
NamespaceID int64
// Namespace
Namespace string `json:"namespace"`
// Repository name
@ -94,3 +96,10 @@ func (c *Candidate) Hash() string {
return base64.StdEncoding.EncodeToString([]byte(raw))
}
// NameHash based on the candidate info for differentiation
func (c *Candidate) NameHash() string {
raw := fmt.Sprintf("%s:%s/%s:%s", c.Kind, c.Namespace, c.Repository, c.Tag)
return base64.StdEncoding.EncodeToString([]byte(raw))
}

12
src/pkg/art/result.go
View File

@ -20,3 +20,15 @@ type Result struct {
// nil error means success
Error error `json:"error"`
}
// ImmutableError ...
type ImmutableError struct {
IsShareDigest bool
}
func (e *ImmutableError) Error() string {
if e.IsShareDigest {
return "Same digest with other immutable tag"
}
return "Immutable tag"
}

1
src/pkg/retention/dep/client.go
View File

@ -106,6 +106,7 @@ func (bc *basicClient) GetCandidates(repository *art.Repository) ([]*art.Candida
}
candidate := &art.Candidate{
Kind: art.Image,
NamespaceID: repository.NamespaceID,
Namespace: repository.Namespace,
Repository: repository.Name,
Tag: image.Name,

18
src/pkg/retention/job.go
View File

@ -32,9 +32,10 @@ import (
)
const (
actionMarkRetain = "RETAIN"
actionMarkDeletion = "DEL"
actionMarkError = "ERR"
actionMarkRetain = "RETAIN"
actionMarkDeletion = "DEL"
actionMarkError = "ERR"
actionMarkImmutable = "IMMUTABLE"
)
// Job of running retention process
@ -116,9 +117,9 @@ func (pj *Job) Run(ctx job.Context, params job.Parameters) error {
return saveRetainNum(ctx, results, allCandidates)
}
func saveRetainNum(ctx job.Context, retained []*art.Result, allCandidates []*art.Candidate) error {
func saveRetainNum(ctx job.Context, results []*art.Result, allCandidates []*art.Candidate) error {
var delNum int
for _, r := range retained {
for _, r := range results {
if r.Error == nil {
delNum++
}
@ -146,9 +147,12 @@ func logResults(logger logger.Interface, all []*art.Candidate, results []*art.Re
}
}
op := func(art *art.Candidate) string {
if e, exists := hash[art.Hash()]; exists {
op := func(c *art.Candidate) string {
if e, exists := hash[c.Hash()]; exists {
if e != nil {
if _, ok := e.(*art.ImmutableError); ok {
return actionMarkImmutable
}
return actionMarkError
}

14
src/pkg/retention/launcher.go
View File

@ -175,9 +175,10 @@ func (l *launcher) Launch(ply *policy.Metadata, executionID int64, isDryRun bool
for _, repositoryCandidate := range repositoryCandidates {
reposit := art.Repository{
Namespace: repositoryCandidate.Namespace,
Name: repositoryCandidate.Repository,
Kind: repositoryCandidate.Kind,
NamespaceID: repositoryCandidate.NamespaceID,
Namespace: repositoryCandidate.Namespace,
Name: repositoryCandidate.Repository,
Kind: repositoryCandidate.Kind,
}
if repositoryRules[reposit] == nil {
repositoryRules[reposit] = &lwp.Metadata{
@ -352,9 +353,10 @@ func getRepositories(projectMgr project.Manager, repositoryMgr repository.Manage
for _, r := range imageRepositories {
namespace, repo := utils.ParseRepository(r.Name)
candidates = append(candidates, &art.Candidate{
Namespace: namespace,
Repository: repo,
Kind: "image",
NamespaceID: projectID,
Namespace: namespace,
Repository: repo,
Kind: "image",
})
}
// currently, doesn't support retention for chart

62
src/pkg/retention/policy/action/performer.go
View File

@ -15,7 +15,10 @@
package action
import (
"github.com/goharbor/harbor/src/common/utils"
"github.com/goharbor/harbor/src/common/utils/log"
"github.com/goharbor/harbor/src/pkg/art"
"github.com/goharbor/harbor/src/pkg/immutabletag/match/rule"
"github.com/goharbor/harbor/src/pkg/retention/dep"
)
@ -50,25 +53,45 @@ type retainAction struct {
// Perform the action
func (ra *retainAction) Perform(candidates []*art.Candidate) (results []*art.Result, err error) {
retained := make(map[string]bool)
immutable := make(map[string]bool)
retainedShare := make(map[string]bool)
immutableShare := make(map[string]bool)
for _, c := range candidates {
retained[c.Hash()] = true
retained[c.NameHash()] = true
retainedShare[c.Hash()] = true
}
for _, c := range ra.all {
if _, ok := retainedShare[c.Hash()]; ok {
continue
}
if isImmutable(c) {
immutable[c.NameHash()] = true
immutableShare[c.Hash()] = true
}
}
// start to delete
if len(ra.all) > 0 {
for _, c := range ra.all {
if _, ok := retained[c.Hash()]; !ok {
result := &art.Result{
Target: c,
}
if !ra.isDryRun {
if err := dep.DefaultClient.Delete(c); err != nil {
result.Error = err
if _, ok := retained[c.NameHash()]; !ok {
if _, ok = retainedShare[c.Hash()]; !ok {
result := &art.Result{
Target: c,
}
if _, ok = immutable[c.NameHash()]; ok {
result.Error = &art.ImmutableError{}
} else if _, ok = immutableShare[c.Hash()]; ok {
result.Error = &art.ImmutableError{IsShareDigest: true}
} else {
if !ra.isDryRun {
if err := dep.DefaultClient.Delete(c); err != nil {
result.Error = err
}
}
}
results = append(results, result)
}
results = append(results, result)
}
}
}
@ -76,6 +99,23 @@ func (ra *retainAction) Perform(candidates []*art.Candidate) (results []*art.Res
return
}
func isImmutable(c *art.Candidate) bool {
projectID := c.NamespaceID
repo := c.Repository
tag := c.Tag
_, repoName := utils.ParseRepository(repo)
matched, err := rule.NewRuleMatcher(projectID).Match(art.Candidate{
Repository: repoName,
Tag: tag,
NamespaceID: projectID,
})
if err != nil {
log.Error(err)
return false
}
return matched
}
// NewRetainAction is factory method for RetainAction
func NewRetainAction(params interface{}, isDryRun bool) Performer {
if params != nil {

120
src/pkg/retention/policy/action/performer_test.go
View File

@ -15,10 +15,13 @@
package action
import (
"github.com/goharbor/harbor/src/common/dao"
"github.com/goharbor/harbor/src/pkg/immutabletag"
"testing"
"time"
"github.com/goharbor/harbor/src/pkg/art"
immumodel "github.com/goharbor/harbor/src/pkg/immutabletag/model"
"github.com/goharbor/harbor/src/pkg/retention/dep"
"github.com/pkg/errors"
"github.com/stretchr/testify/assert"
@ -64,6 +67,7 @@ func (suite *TestPerformerSuite) SetupSuite() {
suite.oldClient = dep.DefaultClient
dep.DefaultClient = &fakeRetentionClient{}
dao.PrepareTestForPostgresSQL()
}
// TearDownSuite ...
@ -97,6 +101,122 @@ func (suite *TestPerformerSuite) TestPerform() {
assert.Equal(suite.T(), "dev", results[0].Target.Tag)
}
// TestPerform tests Perform action
func (suite *TestPerformerSuite) TestPerformImmutable() {
all := []*art.Candidate{
{
NamespaceID: 1,
Namespace: "library",
Repository: "harbor",
Kind: "image",
Tag: "latest",
Digest: "d0",
PushedTime: time.Now().Unix(),
Labels: []string{"L1", "L2"},
},
{
NamespaceID: 1,
Namespace: "library",
Repository: "harbor",
Kind: "image",
Tag: "dev",
Digest: "d1",
PushedTime: time.Now().Unix(),
Labels: []string{"L3"},
},
{
NamespaceID: 1,
Namespace: "library",
Repository: "test",
Kind: "image",
Tag: "immute",
Digest: "d2",
PushedTime: time.Now().Unix(),
Labels: []string{"L1", "L2"},
},
{
NamespaceID: 1,
Namespace: "library",
Repository: "test",
Kind: "image",
Tag: "samedig",
Digest: "d2",
PushedTime: time.Now().Unix(),
Labels: []string{"L1", "L2"},
},
}
p := &retainAction{
all: all,
}
rule := &immumodel.Metadata{
ProjectID: 1,
Priority: 1,
Action: "immutable",
Template: "immutable_template",
TagSelectors: []*immumodel.Selector{
{
Kind: "doublestar",
Decoration: "matches",
Pattern: "immute",
},
},
ScopeSelectors: map[string][]*immumodel.Selector{
"repository": {
{
Kind: "doublestar",
Decoration: "repoMatches",
Pattern: "**",
},
},
},
}
imid, e := immutabletag.ImmuCtr.CreateImmutableRule(rule)
assert.NoError(suite.T(), e)
defer func() {
assert.NoError(suite.T(), immutabletag.ImmuCtr.DeleteImmutableRule(imid))
}()
candidates := []*art.Candidate{
{
NamespaceID: 1,
Namespace: "library",
Repository: "harbor",
Kind: "image",
Tag: "latest",
Digest: "d0",
PushedTime: time.Now().Unix(),
Labels: []string{"L1", "L2"},
},
}
results, err := p.Perform(candidates)
require.NoError(suite.T(), err)
require.Equal(suite.T(), 3, len(results))
for _, r := range results {
require.NotNil(suite.T(), r.Target)
if r.Target.Digest == "d1" {
require.NoError(suite.T(), r.Error)
require.Equal(suite.T(), "dev", r.Target.Tag)
} else if r.Target.Digest == "d2" {
require.Error(suite.T(), r.Error)
require.IsType(suite.T(), (*art.ImmutableError)(nil), r.Error)
if i, ok := r.Error.(*art.ImmutableError); ok {
if r.Target.Tag == "immute" {
require.False(suite.T(), i.IsShareDigest)
} else {
require.True(suite.T(), i.IsShareDigest)
}
}
} else {
require.Fail(suite.T(), "should not delete "+r.Target.NameHash())
}
}
require.NotNil(suite.T(), results[0].Target)
assert.NoError(suite.T(), results[0].Error)
assert.Equal(suite.T(), "dev", results[0].Target.Tag)
}
type fakeRetentionClient struct{}
// GetCandidates ...

2
src/pkg/retention/policy/alg/or/processor_test.go
View File

@ -16,6 +16,7 @@ package or
import (
"errors"
"github.com/goharbor/harbor/src/common/dao"
"testing"
"time"
@ -50,6 +51,7 @@ func TestProcessor(t *testing.T) {
// SetupSuite ...
func (suite *ProcessorTestSuite) SetupSuite() {
dao.PrepareTestForPostgresSQL()
suite.all = []*art.Candidate{
{
Namespace: "library",

2
src/pkg/retention/policy/builder_test.go
View File

@ -15,6 +15,7 @@
package policy
import (
"github.com/goharbor/harbor/src/common/dao"
"testing"
"time"
@ -66,6 +67,7 @@ func TestBuilder(t *testing.T) {
// SetupSuite prepares the testing content if needed
func (suite *TestBuilderSuite) SetupSuite() {
dao.PrepareTestForPostgresSQL()
suite.all = []*art.Candidate{
{
NamespaceID: 1,

2
src/pkg/retention/policy/lwp/models.go
View File

@ -25,6 +25,8 @@ import (
// Metadata contains partial metadata of policy
// It's a lightweight version of policy.Metadata
type Metadata struct {
// ID of the policy
ID int64 `json:"id"`
// Algorithm applied to the rules
// "OR" / "AND"
Algorithm string `json:"algorithm"`