Merge pull request #8317 from steven-zou/feature/retention-selectors

implement label and regexp selectors

src/pkg/retention/launcher_test.go

@@ -27,7 +27,7 @@ import (
 	"github.com/goharbor/harbor/src/pkg/retention/policy/rule"
 	"github.com/goharbor/harbor/src/pkg/retention/q"
 	"github.com/goharbor/harbor/src/pkg/retention/res"
-	_ "github.com/goharbor/harbor/src/pkg/retention/res/selectors/regexp"
+	_ "github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"

src/pkg/retention/policy/alg/or/processor_test.go

@@ -21,8 +21,8 @@ import (
 	"github.com/goharbor/harbor/src/pkg/retention/policy/rule/lastx"
 	"github.com/goharbor/harbor/src/pkg/retention/policy/rule/latestk"
 	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/goharbor/harbor/src/pkg/retention/res/selectors/doublestar"
 	"github.com/goharbor/harbor/src/pkg/retention/res/selectors/label"
-	"github.com/goharbor/harbor/src/pkg/retention/res/selectors/regexp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
@@ -73,7 +73,7 @@ func (suite *ProcessorTestSuite) SetupSuite() {
 	params = append(params, &alg.Parameter{
 		Evaluator: lastx.New(lastxParams),
 		Selectors: []res.Selector{
-			regexp.New(regexp.Matches, "*dev*"),
+			doublestar.New(doublestar.Matches, "*dev*"),
 			label.New(label.With, "L1,L2"),
 		},
 		Performer: perf,

src/pkg/retention/res/selectors/doublestar/selector.go

@@ -0,0 +1,99 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package doublestar
+
+import (
+	"github.com/bmatcuk/doublestar"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
+)
+
+const (
+	// Kind ...
+	Kind = "doublestar"
+	// Matches [pattern] for tag (default)
+	Matches = "matches"
+	// Excludes [pattern] for tag (default)
+	Excludes = "excludes"
+	// RepoMatches represents repository matches [pattern]
+	RepoMatches = "repoMatches"
+	// RepoExcludes represents repository excludes [pattern]
+	RepoExcludes = "repoExcludes"
+)
+
+// selector for regular expression
+type selector struct {
+	// Pre defined pattern declarator
+	// "matches", "excludes", "repoMatches" or "repoExcludes"
+	decoration string
+	// The pattern expression
+	pattern string
+}
+
+// Select candidates by regular expressions
+func (s *selector) Select(artifacts []*res.Candidate) (selected []*res.Candidate, err error) {
+	value := ""
+	excludes := false
+
+	for _, art := range artifacts {
+		switch s.decoration {
+		case Matches:
+			value = art.Tag
+		case Excludes:
+			value = art.Tag
+			excludes = true
+		case RepoMatches:
+			value = art.Repository
+		case RepoExcludes:
+			value = art.Repository
+			excludes = true
+		}
+
+		if len(value) > 0 {
+			matched, err := match(s.pattern, value)
+			if err != nil {
+				// if error occurred, directly throw it out
+				return nil, err
+			}
+
+			if (matched && !excludes) || (!matched && excludes) {
+				selected = append(selected, art)
+			}
+		}
+	}
+
+	return selected, nil
+}
+
+// New is factory method for doublestar selector
+func New(decoration string, pattern string) res.Selector {
+	return &selector{
+		decoration: decoration,
+		pattern:    pattern,
+	}
+}
+
+// match returns whether the str matches the pattern
+func match(pattern, str string) (bool, error) {
+	if len(pattern) == 0 {
+		return true, nil
+	}
+	return doublestar.Match(pattern, str)
+}
+
+func init() {
+	// Register doublestar selector
+	selectors.Register(Kind, []string{Matches, Excludes}, New)
+}

src/pkg/retention/res/selectors/doublestar/selector_test.go

@@ -0,0 +1,198 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package doublestar
+
+import (
+	"fmt"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"testing"
+	"time"
+)
+
+// RegExpSelectorTestSuite is a suite for testing the label selector
+type RegExpSelectorTestSuite struct {
+	suite.Suite
+
+	artifacts []*res.Candidate
+}
+
+// TestRegExpSelector is entrance for RegExpSelectorTestSuite
+func TestRegExpSelector(t *testing.T) {
+	suite.Run(t, new(RegExpSelectorTestSuite))
+}
+
+// SetupSuite to do preparation work
+func (suite *RegExpSelectorTestSuite) SetupSuite() {
+	suite.artifacts = []*res.Candidate{
+		{
+			NamespaceID:  1,
+			Namespace:    "library",
+			Repository:   "harbor",
+			Tag:          "latest",
+			Kind:         res.Image,
+			PushedTime:   time.Now().Unix() - 3600,
+			PulledTime:   time.Now().Unix(),
+			CreationTime: time.Now().Unix() - 7200,
+			Labels:       []string{"label1", "label2", "label3"},
+		},
+		{
+			NamespaceID:  1,
+			Namespace:    "library",
+			Repository:   "redis",
+			Tag:          "4.0",
+			Kind:         res.Image,
+			PushedTime:   time.Now().Unix() - 3600,
+			PulledTime:   time.Now().Unix(),
+			CreationTime: time.Now().Unix() - 7200,
+			Labels:       []string{"label1", "label4", "label5"},
+		},
+		{
+			NamespaceID:  1,
+			Namespace:    "library",
+			Repository:   "redis",
+			Tag:          "4.1",
+			Kind:         res.Image,
+			PushedTime:   time.Now().Unix() - 3600,
+			PulledTime:   time.Now().Unix(),
+			CreationTime: time.Now().Unix() - 7200,
+			Labels:       []string{"label1", "label4", "label5"},
+		},
+	}
+}
+
+// TestTagMatches tests the tag `matches` case
+func (suite *RegExpSelectorTestSuite) TestTagMatches() {
+	tagMatches := &selector{
+		decoration: Matches,
+		pattern:    "{latest,4.*}",
+	}
+
+	selected, err := tagMatches.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 3, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"harbor:latest", "redis:4.0", "redis:4.1"}, selected)
+	})
+
+	tagMatches2 := &selector{
+		decoration: Matches,
+		pattern:    "4.*",
+	}
+
+	selected, err = tagMatches2.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 2, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"redis:4.0", "redis:4.1"}, selected)
+	})
+}
+
+// TestTagExcludes tests the tag `excludes` case
+func (suite *RegExpSelectorTestSuite) TestTagExcludes() {
+	tagExcludes := &selector{
+		decoration: Excludes,
+		pattern:    "{latest,4.*}",
+	}
+
+	selected, err := tagExcludes.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 0, len(selected))
+
+	tagExcludes2 := &selector{
+		decoration: Excludes,
+		pattern:    "4.*",
+	}
+
+	selected, err = tagExcludes2.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 1, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"harbor:latest"}, selected)
+	})
+}
+
+// TestRepoMatches tests the repository `matches` case
+func (suite *RegExpSelectorTestSuite) TestRepoMatches() {
+	repoMatches := &selector{
+		decoration: RepoMatches,
+		pattern:    "{redis}",
+	}
+
+	selected, err := repoMatches.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 2, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"redis:4.0", "redis:4.1"}, selected)
+	})
+
+	repoMatches2 := &selector{
+		decoration: RepoMatches,
+		pattern:    "har*",
+	}
+
+	selected, err = repoMatches2.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 1, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"harbor:latest"}, selected)
+	})
+}
+
+// TestRepoExcludes tests the repository `excludes` case
+func (suite *RegExpSelectorTestSuite) TestRepoExcludes() {
+	repoExcludes := &selector{
+		decoration: RepoExcludes,
+		pattern:    "{redis}",
+	}
+
+	selected, err := repoExcludes.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 1, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"harbor:latest"}, selected)
+	})
+
+	repoExcludes2 := &selector{
+		decoration: RepoExcludes,
+		pattern:    "har*",
+	}
+
+	selected, err = repoExcludes2.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 2, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"redis:4.0", "redis:4.1"}, selected)
+	})
+}
+
+// Check whether the returned result matched the expected ones (only check repo:tag)
+func expect(expected []string, candidates []*res.Candidate) bool {
+	hash := make(map[string]bool)
+
+	for _, art := range candidates {
+		hash[fmt.Sprintf("%s:%s", art.Repository, art.Tag)] = true
+	}
+
+	for _, exp := range expected {
+		if _, ok := hash[exp]; !ok {
+			return ok
+		}
+	}
+
+	return true
+}

src/pkg/retention/res/selectors/label/selector.go

@@ -38,10 +38,15 @@ type selector struct {
 	labels []string
 }
 
-// Select candidates by regular expressions
-func (s *selector) Select(artifacts []*res.Candidate) ([]*res.Candidate, error) {
-	// TODO: REPLACE SAMPLE CODE WITH REAL IMPLEMENTATION
-	return artifacts, nil
+// Select candidates by the labels
+func (s *selector) Select(artifacts []*res.Candidate) (selected []*res.Candidate, err error) {
+	for _, art := range artifacts {
+		if isMatched(s.labels, art.Labels, s.decoration) {
+			selected = append(selected, art)
+		}
+	}
+
+	return selected, nil
 }
 
 // New is factory method for list selector
@@ -54,7 +59,30 @@ func New(decoration string, pattern string) res.Selector {
 	}
 }
 
+// Check if the resource labels match the pattern labels
+func isMatched(patternLbls []string, resLbls []string, decoration string) bool {
+	hash := make(map[string]bool)
+
+	for _, lbl := range resLbls {
+		hash[lbl] = true
+	}
+
+	for _, lbl := range patternLbls {
+		_, exists := hash[lbl]
+
+		if decoration == Without && exists {
+			return false
+		}
+
+		if decoration == With && !exists {
+			return false
+		}
+	}
+
+	return true
+}
+
 func init() {
-	// Register regexp selector
+	// Register doublestar selector
 	selectors.Register(Kind, []string{With, Without}, New)
 }

src/pkg/retention/res/selectors/label/selector_test.go

@@ -0,0 +1,148 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package label
+
+import (
+	"fmt"
+	"github.com/goharbor/harbor/src/pkg/retention/res"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+	"testing"
+	"time"
+)
+
+// LabelSelectorTestSuite is a suite for testing the label selector
+type LabelSelectorTestSuite struct {
+	suite.Suite
+
+	artifacts []*res.Candidate
+}
+
+// TestLabelSelector is entrance for LabelSelectorTestSuite
+func TestLabelSelector(t *testing.T) {
+	suite.Run(t, new(LabelSelectorTestSuite))
+}
+
+// SetupSuite to do preparation work
+func (suite *LabelSelectorTestSuite) SetupSuite() {
+	suite.artifacts = []*res.Candidate{
+		{
+			NamespaceID:  1,
+			Namespace:    "library",
+			Repository:   "harbor",
+			Tag:          "1.9",
+			Kind:         res.Image,
+			PushedTime:   time.Now().Unix() - 3600,
+			PulledTime:   time.Now().Unix(),
+			CreationTime: time.Now().Unix() - 7200,
+			Labels:       []string{"label1", "label2", "label3"},
+		},
+		{
+			NamespaceID:  1,
+			Namespace:    "library",
+			Repository:   "harbor",
+			Tag:          "dev",
+			Kind:         res.Image,
+			PushedTime:   time.Now().Unix() - 3600,
+			PulledTime:   time.Now().Unix(),
+			CreationTime: time.Now().Unix() - 7200,
+			Labels:       []string{"label1", "label4", "label5"},
+		},
+	}
+}
+
+// TestWithLabelsUnMatched tests the selector of `with` labels but nothing matched
+func (suite *LabelSelectorTestSuite) TestWithLabelsUnMatched() {
+	withNothing := &selector{
+		decoration: With,
+		labels:     []string{"label6"},
+	}
+
+	selected, err := withNothing.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 0, len(selected))
+}
+
+// TestWithLabelsMatched tests the selector of `with` labels and matched something
+func (suite *LabelSelectorTestSuite) TestWithLabelsMatched() {
+	with1 := &selector{
+		decoration: With,
+		labels:     []string{"label2"},
+	}
+
+	selected, err := with1.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 1, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"harbor:1.9"}, selected)
+	})
+
+	with2 := &selector{
+		decoration: With,
+		labels:     []string{"label1"},
+	}
+
+	selected2, err := with2.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 2, len(selected2))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"harbor:1.9", "harbor:dev"}, selected2)
+	})
+}
+
+// TestWithoutExistingLabels tests the selector of `without` existing labels
+func (suite *LabelSelectorTestSuite) TestWithoutExistingLabels() {
+	withoutExisting := &selector{
+		decoration: Without,
+		labels:     []string{"label1"},
+	}
+
+	selected, err := withoutExisting.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 0, len(selected))
+}
+
+// TestWithoutNoneExistingLabels tests the selector of `without` non-existing labels
+func (suite *LabelSelectorTestSuite) TestWithoutNoneExistingLabels() {
+	withoutNonExisting := &selector{
+		decoration: Without,
+		labels:     []string{"label6"},
+	}
+
+	selected, err := withoutNonExisting.Select(suite.artifacts)
+	require.NoError(suite.T(), err)
+	assert.Equal(suite.T(), 2, len(selected))
+	assert.Condition(suite.T(), func() bool {
+		return expect([]string{"harbor:1.9", "harbor:dev"}, selected)
+	})
+}
+
+// Check whether the returned result matched the expected ones (only check repo:tag)
+func expect(expected []string, candidates []*res.Candidate) bool {
+	hash := make(map[string]bool)
+
+	for _, art := range candidates {
+		hash[fmt.Sprintf("%s:%s", art.Repository, art.Tag)] = true
+	}
+
+	for _, exp := range expected {
+		if _, ok := hash[exp]; !ok {
+			return ok
+		}
+	}
+
+	return true
+}

src/pkg/retention/res/selectors/regexp/selector.go

@@ -1,57 +0,0 @@
-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package regexp
-
-import (
-	"github.com/goharbor/harbor/src/pkg/retention/res"
-	"github.com/goharbor/harbor/src/pkg/retention/res/selectors"
-)
-
-const (
-	// Kind ...
-	Kind = "regularExpression"
-	// Matches [pattern]
-	Matches = "matches"
-	// Excludes [pattern]
-	Excludes = "excludes"
-)
-
-// selector for regular expression
-type selector struct {
-	// Pre defined pattern declarator
-	// "matches" and "excludes"
-	decoration string
-	// The pattern expression
-	pattern string
-}
-
-// Select candidates by regular expressions
-func (s *selector) Select(artifacts []*res.Candidate) ([]*res.Candidate, error) {
-	// TODO: REPLACE SAMPLE CODE WITH REAL IMPLEMENTATION
-	return artifacts, nil
-}
-
-// New is factory method for regexp selector
-func New(decoration string, pattern string) res.Selector {
-	return &selector{
-		decoration: decoration,
-		pattern:    pattern,
-	}
-}
-
-func init() {
-	// Register regexp selector
-	selectors.Register(Kind, []string{Matches, Excludes}, New)
-}