feat(replication) add exclude decoration in policy

Signed-off-by: Ziming Zhang <zziming@vmware.com>

api/v2.0/swagger.yaml

@@ -6477,6 +6477,9 @@ definitions:
       value:
         type: object
         description: 'The value of replication policy filter.'
+      decoration:
+        type: string
+        description: 'matches or excludes the result'
   RegistryCredential:
     type: object
     properties:

src/controller/replication/model/model.go

@@ -94,6 +94,12 @@ func (p *Policy) Validate() error {
 						WithMessage("invalid resource filter: %s", value)
 				}
 			}
+			if filter.Type == model.FilterTypeName || filter.Type == model.FilterTypeResource {
+				if filter.Decoration != "" {
+					return errors.New(nil).WithCode(errors.BadRequestCode).
+						WithMessage("only tag and label filter support decoration")
+				}
+			}
 		case model.FilterTypeLabel:
 			labels, ok := filter.Value.([]interface{})
 			if !ok {
@@ -111,6 +117,11 @@ func (p *Policy) Validate() error {
 			return errors.New(nil).WithCode(errors.BadRequestCode).
 				WithMessage("invalid filter type")
 		}
+
+		if filter.Decoration != "" && filter.Decoration != model.Matches && filter.Decoration != model.Excludes {
+			return errors.New(nil).WithCode(errors.BadRequestCode).
+				WithMessage("invalid filter decoration, :%s", filter.Decoration)
+		}
 	}
 
 	// valid the destination namespace
@@ -231,6 +242,7 @@ func (p *Policy) To() (*replicationmodel.Policy, error) {
 type filter struct {
 	Type       string      `json:"type"`
 	Value      interface{} `json:"value"`
+	Decoration string      `json:"decoration"`
 	Kind       string      `json:"kind"`
 	Pattern    string      `json:"pattern"`
 }
@@ -262,6 +274,7 @@ func parseFilters(str string) ([]*model.Filter, error) {
 		filter := &model.Filter{
 			Type:       item.Type,
 			Value:      item.Value,
+			Decoration: item.Decoration,
 		}
 		// keep backwards compatibility
 		if len(filter.Type) == 0 {

src/pkg/reg/adapter/aliacr/adapter.go

@@ -18,6 +18,7 @@ import (
 	"github.com/goharbor/harbor/src/lib/log"
 	adp "github.com/goharbor/harbor/src/pkg/reg/adapter"
 	"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
+	"github.com/goharbor/harbor/src/pkg/reg/filter"
 	"github.com/goharbor/harbor/src/pkg/reg/model"
 	"github.com/goharbor/harbor/src/pkg/reg/util"
 	"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
@@ -236,12 +237,9 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Re
 	// get filter pattern
 	var repoPattern string
 	var tagsPattern string
-	for _, filter := range filters {
+	for _, f := range filters {
-		if filter.Type == model.FilterTypeName {
+		if f.Type == model.FilterTypeName {
-			repoPattern = filter.Value.(string)
+			repoPattern = f.Value.(string)
-		}
-		if filter.Type == model.FilterTypeTag {
-			tagsPattern = filter.Value.(string)
 		}
 	}
 	var namespacePattern = strings.Split(repoPattern, "/")[0]
@@ -295,23 +293,18 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Re
 				return fmt.Errorf("list tags for repo '%s' error: %v", repo.RepoName, err)
 			}
 
-			var filterTags []string
+			var artifacts []*model.Artifact
-			if tagsPattern != "" {
 			for _, tag := range tags {
-					var ok bool
+				artifacts = append(artifacts, &model.Artifact{
-					ok, err = util.Match(tagsPattern, tag)
+					Tags: []string{tag},
+				})
+			}
+			filterArtifacts, err := filter.DoFilterArtifacts(artifacts, filters)
 			if err != nil {
-						return fmt.Errorf("match tag '%s' error: %v", tag, err)
+				return err
-					}
-					if ok {
-						filterTags = append(filterTags, tag)
-					}
-				}
-			} else {
-				filterTags = tags
 			}
 
-			if len(filterTags) > 0 {
+			if len(filterArtifacts) > 0 {
 				rawResources[index] = &model.Resource{
 					Type:     model.ResourceTypeImage,
 					Registry: a.registry,
@@ -319,7 +312,7 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Re
 						Repository: &model.Repository{
 							Name: filepath.Join(repo.RepoNamespace, repo.RepoName),
 						},
-						Vtags: filterTags,
+						Artifacts: filterArtifacts,
 					},
 				}
 			}

src/pkg/reg/adapter/dockerhub/adapter.go

@@ -13,6 +13,7 @@ import (
 	"github.com/goharbor/harbor/src/lib/log"
 	adp "github.com/goharbor/harbor/src/pkg/reg/adapter"
 	"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
+	"github.com/goharbor/harbor/src/pkg/reg/filter"
 	"github.com/goharbor/harbor/src/pkg/reg/model"
 	"github.com/goharbor/harbor/src/pkg/reg/util"
 )
@@ -245,10 +246,6 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
 	if err != nil {
 		return nil, err
 	}
-	tagFilter, err := a.getStringFilterValue(model.FilterTypeTag, filters)
-	if err != nil {
-		return nil, err
-	}
 
 	namespaces, err := a.listCandidateNamespaces(nameFilter)
 	if err != nil {
@@ -305,17 +302,6 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
 					return fmt.Errorf("get tags for repo '%s/%s' from DockerHub error: %v", repo.Namespace, repo.Name, err)
 				}
 				for _, t := range pageTags.Tags {
-					// If tag filter set, skip tags that don't match the filter pattern.
-					if len(tagFilter) != 0 {
-						m, err := util.Match(tagFilter, t.Name)
-						if err != nil {
-							return fmt.Errorf("match tag name '%s' against pattern '%s' error: %v", t.Name, tagFilter, err)
-						}
-
-						if !m {
-							continue
-						}
-					}
 					tags = append(tags, t.Name)
 				}
 
@@ -325,6 +311,17 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
 				page++
 			}
 
+			var artifacts []*model.Artifact
+			for _, tag := range tags {
+				artifacts = append(artifacts, &model.Artifact{
+					Tags: []string{tag},
+				})
+			}
+			filterArtifacts, err := filter.DoFilterArtifacts(artifacts, filters)
+			if err != nil {
+				return err
+			}
+
 			if len(tags) > 0 {
 				rawResources[index] = &model.Resource{
 					Type:     model.ResourceTypeImage,
@@ -333,7 +330,7 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
 						Repository: &model.Repository{
 							Name: name,
 						},
-						Vtags: tags,
+						Artifacts: filterArtifacts,
 					},
 				}
 			}

src/pkg/reg/filter/artifact.go

@@ -38,16 +38,18 @@ func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
 		case model.FilterTypeLabel:
 			f = &artifactLabelFilter{
 				labels:     filter.Value.([]string),
+				decoration: filter.Decoration,
 			}
 		case model.FilterTypeTag:
 			f = &artifactTagFilter{
 				pattern:    filter.Value.(string),
+				decoration: filter.Decoration,
 			}
 		case model.FilterTypeResource:
 			v := filter.Value.(string)
 			if v != model.ResourceTypeArtifact && v != model.ResourceTypeChart {
 				f = &artifactTypeFilter{
-					types: []string{string(v)},
+					types: []string{v},
 				}
 			}
 		}
@@ -102,6 +104,8 @@ func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artif
 // in the filter is the valid one
 type artifactLabelFilter struct {
 	labels []string
+	// "matches", "excludes"
+	decoration string
 }
 
 func (a *artifactLabelFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
@@ -122,10 +126,16 @@ func (a *artifactLabelFilter) Filter(artifacts []*model.Artifact) ([]*model.Arti
 			}
 		}
 		// add the artifact to the result list if it contains all labels defined for the filter
+		if a.decoration == model.Excludes {
+			if !match {
+				result = append(result, artifact)
+			}
+		} else {
 			if match {
 				result = append(result, artifact)
 			}
 		}
+	}
 	return result, nil
 }
 
@@ -147,6 +157,8 @@ func (a *artifactTaggedFilter) Filter(artifacts []*model.Artifact) ([]*model.Art
 
 type artifactTagFilter struct {
 	pattern string
+	// "matches", "excludes"
+	decoration string
 }
 
 func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
@@ -161,9 +173,15 @@ func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifa
 			if err != nil {
 				return nil, err
 			}
+			if a.decoration == model.Excludes {
+				if !match {
+					result = append(result, artifact)
+				}
+			} else {
 				if match {
 					result = append(result, artifact)
 				}
+			}
 			continue
 		}
 
@@ -174,9 +192,14 @@ func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifa
 			if err != nil {
 				return nil, err
 			}
+			if a.decoration == model.Excludes {
+				if !match {
+					tags = append(tags, tag)
+				}
+			} else {
 				if match {
 					tags = append(tags, tag)
-				continue
+				}
 			}
 		}
 		if len(tags) == 0 {

src/pkg/reg/filter/resource_test.go

@@ -0,0 +1,161 @@
+package filter
+
+import (
+	"github.com/goharbor/harbor/src/pkg/reg/model"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestArtifactTagFilters(t *testing.T) {
+	var artifacts = []*model.Artifact{
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "aaaaa",
+			Tags: []string{
+				"test1",
+				"test2",
+				"harbor1",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "bbbbb",
+			Tags: []string{
+				"test3",
+				"harbor2",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "ccccc",
+			Tags: []string{
+				"harbor3",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "ddddd",
+		},
+	}
+
+	var filters = []*model.Filter{
+		{
+			Type:  model.FilterTypeTag,
+			Value: "test*",
+		},
+	}
+
+	artFilters, err := BuildArtifactFilters(filters)
+	require.Nil(t, err)
+
+	arts, err := artFilters.Filter(artifacts)
+	require.Nil(t, err)
+	require.Equal(t, 2, len(arts))
+	require.EqualValues(t, "aaaaa", arts[0].Digest)
+	require.EqualValues(t, []string{"test1", "test2"}, arts[0].Tags)
+	require.EqualValues(t, "bbbbb", arts[1].Digest)
+	require.EqualValues(t, []string{"test3"}, arts[1].Tags)
+
+	filters = []*model.Filter{
+		{
+			Type:       model.FilterTypeTag,
+			Value:      "test*",
+			Decoration: model.Excludes,
+		},
+	}
+
+	artFilters, err = BuildArtifactFilters(filters)
+	require.Nil(t, err)
+
+	arts, err = artFilters.Filter(artifacts)
+	require.Nil(t, err)
+	require.Equal(t, 4, len(arts))
+	require.EqualValues(t, "aaaaa", arts[0].Digest)
+	require.EqualValues(t, []string{"harbor1"}, arts[0].Tags)
+	require.EqualValues(t, "bbbbb", arts[1].Digest)
+	require.EqualValues(t, []string{"harbor2"}, arts[1].Tags)
+	require.EqualValues(t, "ccccc", arts[2].Digest)
+	require.EqualValues(t, []string{"harbor3"}, arts[2].Tags)
+	require.EqualValues(t, "ddddd", arts[3].Digest)
+	require.Nil(t, arts[3].Tags)
+}
+
+func TestArtifactLabelFilters(t *testing.T) {
+	var artifacts = []*model.Artifact{
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "aaaaa",
+			Tags: []string{
+				"test1",
+				"test2",
+				"harbor1",
+			},
+			Labels: []string{
+				"label1",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "bbbbb",
+			Tags: []string{
+				"test3",
+				"harbor2",
+			},
+			Labels: []string{
+				"label1",
+				"label2",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "ccccc",
+			Tags: []string{
+				"harbor3",
+			},
+			Labels: []string{
+				"label3",
+			},
+		},
+		{
+			Type:   model.ResourceTypeArtifact,
+			Digest: "ddddd",
+		},
+	}
+
+	var filters = []*model.Filter{
+		{
+			Type:  model.FilterTypeLabel,
+			Value: []string{"label1"},
+		},
+	}
+
+	artFilters, err := BuildArtifactFilters(filters)
+	require.Nil(t, err)
+
+	arts, err := artFilters.Filter(artifacts)
+	require.Nil(t, err)
+	require.Equal(t, 2, len(arts))
+	require.EqualValues(t, "aaaaa", arts[0].Digest)
+	require.EqualValues(t, []string{"label1"}, arts[0].Labels)
+	require.EqualValues(t, "bbbbb", arts[1].Digest)
+	require.EqualValues(t, []string{"label1", "label2"}, arts[1].Labels)
+
+	filters = []*model.Filter{
+		{
+			Type:       model.FilterTypeLabel,
+			Value:      []string{"label1"},
+			Decoration: model.Excludes,
+		},
+	}
+
+	artFilters, err = BuildArtifactFilters(filters)
+	require.Nil(t, err)
+
+	arts, err = artFilters.Filter(artifacts)
+	require.Nil(t, err)
+	require.Equal(t, 2, len(arts))
+	require.EqualValues(t, "ccccc", arts[0].Digest)
+	require.EqualValues(t, []string{"label3"}, arts[0].Labels)
+	require.EqualValues(t, "ddddd", arts[1].Digest)
+	require.Nil(t, arts[1].Labels)
+}

src/pkg/reg/model/policy.go

@@ -24,12 +24,18 @@ const (
 	TriggerTypeManual     = "manual"
 	TriggerTypeScheduled  = "scheduled"
 	TriggerTypeEventBased = "event_based"
+
+	// Matches [pattern] for tag (default)
+	Matches = "matches"
+	// Excludes [pattern] for tag
+	Excludes = "excludes"
 )
 
 // Filter holds the info of the filter
 type Filter struct {
 	Type       string      `json:"type"`
 	Value      interface{} `json:"value"`
+	Decoration string      `json:"decoration,omitempty"`
 }
 
 // Trigger holds info for a trigger

src/pkg/reg/util/pattern_test.go

@@ -33,6 +33,11 @@ func TestMatch(t *testing.T) {
 			str:     "library",
 			match:   true,
 		},
+		{
+			pattern: "",
+			str:     "",
+			match:   true,
+		},
 		{
 			pattern: "*",
 			str:     "library",

src/server/v2.0/handler/replication.go

@@ -87,6 +87,7 @@ func (r *replicationAPI) CreateReplicationPolicy(ctx context.Context, params ope
 			policy.Filters = append(policy.Filters, &model.Filter{
 				Type:       filter.Type,
 				Value:      filter.Value,
+				Decoration: filter.Decoration,
 			})
 		}
 	}
@@ -143,6 +144,7 @@ func (r *replicationAPI) UpdateReplicationPolicy(ctx context.Context, params ope
 			policy.Filters = append(policy.Filters, &model.Filter{
 				Type:       filter.Type,
 				Value:      filter.Value,
+				Decoration: filter.Decoration,
 			})
 		}
 	}
@@ -425,6 +427,7 @@ func convertReplicationPolicy(policy *repctlmodel.Policy) *models.ReplicationPol
 			p.Filters = append(p.Filters, &models.ReplicationFilter{
 				Type:       string(filter.Type),
 				Value:      filter.Value,
+				Decoration: filter.Decoration,
 			})
 		}
 	}