From 698c3364213365d6640ae35906fae1a1fd8d4771 Mon Sep 17 00:00:00 2001 From: Ziming Zhang Date: Wed, 21 Jul 2021 10:59:55 +0800 Subject: [PATCH] feat(replication) add exclude decoration in policy Signed-off-by: Ziming Zhang --- api/v2.0/swagger.yaml | 3 + src/controller/replication/model/model.go | 25 +++- src/pkg/reg/adapter/aliacr/adapter.go | 37 ++--- src/pkg/reg/adapter/dockerhub/adapter.go | 29 ++-- src/pkg/reg/filter/artifact.go | 43 ++++-- src/pkg/reg/filter/resource_test.go | 161 ++++++++++++++++++++++ src/pkg/reg/model/policy.go | 10 +- src/pkg/reg/util/pattern_test.go | 5 + src/server/v2.0/handler/replication.go | 15 +- 9 files changed, 266 insertions(+), 62 deletions(-) create mode 100644 src/pkg/reg/filter/resource_test.go diff --git a/api/v2.0/swagger.yaml b/api/v2.0/swagger.yaml index 049e9222a..e67b967db 100644 --- a/api/v2.0/swagger.yaml +++ b/api/v2.0/swagger.yaml @@ -6477,6 +6477,9 @@ definitions: value: type: object description: 'The value of replication policy filter.' + decoration: + type: string + description: 'matches or excludes the result' RegistryCredential: type: object properties: diff --git a/src/controller/replication/model/model.go b/src/controller/replication/model/model.go index 495a048f7..a0ef038b4 100644 --- a/src/controller/replication/model/model.go +++ b/src/controller/replication/model/model.go @@ -94,6 +94,12 @@ func (p *Policy) Validate() error { WithMessage("invalid resource filter: %s", value) } } + if filter.Type == model.FilterTypeName || filter.Type == model.FilterTypeResource { + if filter.Decoration != "" { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("only tag and label filter support decoration") + } + } case model.FilterTypeLabel: labels, ok := filter.Value.([]interface{}) if !ok { @@ -111,6 +117,11 @@ func (p *Policy) Validate() error { return errors.New(nil).WithCode(errors.BadRequestCode). WithMessage("invalid filter type") } + + if filter.Decoration != "" && filter.Decoration != model.Matches && filter.Decoration != model.Excludes { + return errors.New(nil).WithCode(errors.BadRequestCode). + WithMessage("invalid filter decoration, :%s", filter.Decoration) + } } // valid the destination namespace @@ -229,10 +240,11 @@ func (p *Policy) To() (*replicationmodel.Policy, error) { } type filter struct { - Type string `json:"type"` - Value interface{} `json:"value"` - Kind string `json:"kind"` - Pattern string `json:"pattern"` + Type string `json:"type"` + Value interface{} `json:"value"` + Decoration string `json:"decoration"` + Kind string `json:"kind"` + Pattern string `json:"pattern"` } type trigger struct { @@ -260,8 +272,9 @@ func parseFilters(str string) ([]*model.Filter, error) { filters := []*model.Filter{} for _, item := range items { filter := &model.Filter{ - Type: item.Type, - Value: item.Value, + Type: item.Type, + Value: item.Value, + Decoration: item.Decoration, } // keep backwards compatibility if len(filter.Type) == 0 { diff --git a/src/pkg/reg/adapter/aliacr/adapter.go b/src/pkg/reg/adapter/aliacr/adapter.go index 83eb8ef1b..0babad361 100644 --- a/src/pkg/reg/adapter/aliacr/adapter.go +++ b/src/pkg/reg/adapter/aliacr/adapter.go @@ -18,6 +18,7 @@ import ( "github.com/goharbor/harbor/src/lib/log" adp "github.com/goharbor/harbor/src/pkg/reg/adapter" "github.com/goharbor/harbor/src/pkg/reg/adapter/native" + "github.com/goharbor/harbor/src/pkg/reg/filter" "github.com/goharbor/harbor/src/pkg/reg/model" "github.com/goharbor/harbor/src/pkg/reg/util" "github.com/goharbor/harbor/src/pkg/registry/auth/bearer" @@ -236,12 +237,9 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Re // get filter pattern var repoPattern string var tagsPattern string - for _, filter := range filters { - if filter.Type == model.FilterTypeName { - repoPattern = filter.Value.(string) - } - if filter.Type == model.FilterTypeTag { - tagsPattern = filter.Value.(string) + for _, f := range filters { + if f.Type == model.FilterTypeName { + repoPattern = f.Value.(string) } } var namespacePattern = strings.Split(repoPattern, "/")[0] @@ -295,23 +293,18 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Re return fmt.Errorf("list tags for repo '%s' error: %v", repo.RepoName, err) } - var filterTags []string - if tagsPattern != "" { - for _, tag := range tags { - var ok bool - ok, err = util.Match(tagsPattern, tag) - if err != nil { - return fmt.Errorf("match tag '%s' error: %v", tag, err) - } - if ok { - filterTags = append(filterTags, tag) - } - } - } else { - filterTags = tags + var artifacts []*model.Artifact + for _, tag := range tags { + artifacts = append(artifacts, &model.Artifact{ + Tags: []string{tag}, + }) + } + filterArtifacts, err := filter.DoFilterArtifacts(artifacts, filters) + if err != nil { + return err } - if len(filterTags) > 0 { + if len(filterArtifacts) > 0 { rawResources[index] = &model.Resource{ Type: model.ResourceTypeImage, Registry: a.registry, @@ -319,7 +312,7 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Re Repository: &model.Repository{ Name: filepath.Join(repo.RepoNamespace, repo.RepoName), }, - Vtags: filterTags, + Artifacts: filterArtifacts, }, } } diff --git a/src/pkg/reg/adapter/dockerhub/adapter.go b/src/pkg/reg/adapter/dockerhub/adapter.go index 4b6df1e69..0ff2fca65 100644 --- a/src/pkg/reg/adapter/dockerhub/adapter.go +++ b/src/pkg/reg/adapter/dockerhub/adapter.go @@ -13,6 +13,7 @@ import ( "github.com/goharbor/harbor/src/lib/log" adp "github.com/goharbor/harbor/src/pkg/reg/adapter" "github.com/goharbor/harbor/src/pkg/reg/adapter/native" + "github.com/goharbor/harbor/src/pkg/reg/filter" "github.com/goharbor/harbor/src/pkg/reg/model" "github.com/goharbor/harbor/src/pkg/reg/util" ) @@ -245,10 +246,6 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er if err != nil { return nil, err } - tagFilter, err := a.getStringFilterValue(model.FilterTypeTag, filters) - if err != nil { - return nil, err - } namespaces, err := a.listCandidateNamespaces(nameFilter) if err != nil { @@ -305,17 +302,6 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er return fmt.Errorf("get tags for repo '%s/%s' from DockerHub error: %v", repo.Namespace, repo.Name, err) } for _, t := range pageTags.Tags { - // If tag filter set, skip tags that don't match the filter pattern. - if len(tagFilter) != 0 { - m, err := util.Match(tagFilter, t.Name) - if err != nil { - return fmt.Errorf("match tag name '%s' against pattern '%s' error: %v", t.Name, tagFilter, err) - } - - if !m { - continue - } - } tags = append(tags, t.Name) } @@ -325,6 +311,17 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er page++ } + var artifacts []*model.Artifact + for _, tag := range tags { + artifacts = append(artifacts, &model.Artifact{ + Tags: []string{tag}, + }) + } + filterArtifacts, err := filter.DoFilterArtifacts(artifacts, filters) + if err != nil { + return err + } + if len(tags) > 0 { rawResources[index] = &model.Resource{ Type: model.ResourceTypeImage, @@ -333,7 +330,7 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er Repository: &model.Repository{ Name: name, }, - Vtags: tags, + Artifacts: filterArtifacts, }, } } diff --git a/src/pkg/reg/filter/artifact.go b/src/pkg/reg/filter/artifact.go index f6be2590b..f9969afe0 100644 --- a/src/pkg/reg/filter/artifact.go +++ b/src/pkg/reg/filter/artifact.go @@ -37,17 +37,19 @@ func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) { switch filter.Type { case model.FilterTypeLabel: f = &artifactLabelFilter{ - labels: filter.Value.([]string), + labels: filter.Value.([]string), + decoration: filter.Decoration, } case model.FilterTypeTag: f = &artifactTagFilter{ - pattern: filter.Value.(string), + pattern: filter.Value.(string), + decoration: filter.Decoration, } case model.FilterTypeResource: v := filter.Value.(string) if v != model.ResourceTypeArtifact && v != model.ResourceTypeChart { f = &artifactTypeFilter{ - types: []string{string(v)}, + types: []string{v}, } } } @@ -102,6 +104,8 @@ func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artif // in the filter is the valid one type artifactLabelFilter struct { labels []string + // "matches", "excludes" + decoration string } func (a *artifactLabelFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) { @@ -122,8 +126,14 @@ func (a *artifactLabelFilter) Filter(artifacts []*model.Artifact) ([]*model.Arti } } // add the artifact to the result list if it contains all labels defined for the filter - if match { - result = append(result, artifact) + if a.decoration == model.Excludes { + if !match { + result = append(result, artifact) + } + } else { + if match { + result = append(result, artifact) + } } } return result, nil @@ -147,6 +157,8 @@ func (a *artifactTaggedFilter) Filter(artifacts []*model.Artifact) ([]*model.Art type artifactTagFilter struct { pattern string + // "matches", "excludes" + decoration string } func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) { @@ -161,8 +173,14 @@ func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifa if err != nil { return nil, err } - if match { - result = append(result, artifact) + if a.decoration == model.Excludes { + if !match { + result = append(result, artifact) + } + } else { + if match { + result = append(result, artifact) + } } continue } @@ -174,9 +192,14 @@ func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifa if err != nil { return nil, err } - if match { - tags = append(tags, tag) - continue + if a.decoration == model.Excludes { + if !match { + tags = append(tags, tag) + } + } else { + if match { + tags = append(tags, tag) + } } } if len(tags) == 0 { diff --git a/src/pkg/reg/filter/resource_test.go b/src/pkg/reg/filter/resource_test.go new file mode 100644 index 000000000..f67f64938 --- /dev/null +++ b/src/pkg/reg/filter/resource_test.go @@ -0,0 +1,161 @@ +package filter + +import ( + "github.com/goharbor/harbor/src/pkg/reg/model" + "github.com/stretchr/testify/require" + "testing" +) + +func TestArtifactTagFilters(t *testing.T) { + var artifacts = []*model.Artifact{ + { + Type: model.ResourceTypeArtifact, + Digest: "aaaaa", + Tags: []string{ + "test1", + "test2", + "harbor1", + }, + }, + { + Type: model.ResourceTypeArtifact, + Digest: "bbbbb", + Tags: []string{ + "test3", + "harbor2", + }, + }, + { + Type: model.ResourceTypeArtifact, + Digest: "ccccc", + Tags: []string{ + "harbor3", + }, + }, + { + Type: model.ResourceTypeArtifact, + Digest: "ddddd", + }, + } + + var filters = []*model.Filter{ + { + Type: model.FilterTypeTag, + Value: "test*", + }, + } + + artFilters, err := BuildArtifactFilters(filters) + require.Nil(t, err) + + arts, err := artFilters.Filter(artifacts) + require.Nil(t, err) + require.Equal(t, 2, len(arts)) + require.EqualValues(t, "aaaaa", arts[0].Digest) + require.EqualValues(t, []string{"test1", "test2"}, arts[0].Tags) + require.EqualValues(t, "bbbbb", arts[1].Digest) + require.EqualValues(t, []string{"test3"}, arts[1].Tags) + + filters = []*model.Filter{ + { + Type: model.FilterTypeTag, + Value: "test*", + Decoration: model.Excludes, + }, + } + + artFilters, err = BuildArtifactFilters(filters) + require.Nil(t, err) + + arts, err = artFilters.Filter(artifacts) + require.Nil(t, err) + require.Equal(t, 4, len(arts)) + require.EqualValues(t, "aaaaa", arts[0].Digest) + require.EqualValues(t, []string{"harbor1"}, arts[0].Tags) + require.EqualValues(t, "bbbbb", arts[1].Digest) + require.EqualValues(t, []string{"harbor2"}, arts[1].Tags) + require.EqualValues(t, "ccccc", arts[2].Digest) + require.EqualValues(t, []string{"harbor3"}, arts[2].Tags) + require.EqualValues(t, "ddddd", arts[3].Digest) + require.Nil(t, arts[3].Tags) +} + +func TestArtifactLabelFilters(t *testing.T) { + var artifacts = []*model.Artifact{ + { + Type: model.ResourceTypeArtifact, + Digest: "aaaaa", + Tags: []string{ + "test1", + "test2", + "harbor1", + }, + Labels: []string{ + "label1", + }, + }, + { + Type: model.ResourceTypeArtifact, + Digest: "bbbbb", + Tags: []string{ + "test3", + "harbor2", + }, + Labels: []string{ + "label1", + "label2", + }, + }, + { + Type: model.ResourceTypeArtifact, + Digest: "ccccc", + Tags: []string{ + "harbor3", + }, + Labels: []string{ + "label3", + }, + }, + { + Type: model.ResourceTypeArtifact, + Digest: "ddddd", + }, + } + + var filters = []*model.Filter{ + { + Type: model.FilterTypeLabel, + Value: []string{"label1"}, + }, + } + + artFilters, err := BuildArtifactFilters(filters) + require.Nil(t, err) + + arts, err := artFilters.Filter(artifacts) + require.Nil(t, err) + require.Equal(t, 2, len(arts)) + require.EqualValues(t, "aaaaa", arts[0].Digest) + require.EqualValues(t, []string{"label1"}, arts[0].Labels) + require.EqualValues(t, "bbbbb", arts[1].Digest) + require.EqualValues(t, []string{"label1", "label2"}, arts[1].Labels) + + filters = []*model.Filter{ + { + Type: model.FilterTypeLabel, + Value: []string{"label1"}, + Decoration: model.Excludes, + }, + } + + artFilters, err = BuildArtifactFilters(filters) + require.Nil(t, err) + + arts, err = artFilters.Filter(artifacts) + require.Nil(t, err) + require.Equal(t, 2, len(arts)) + require.EqualValues(t, "ccccc", arts[0].Digest) + require.EqualValues(t, []string{"label3"}, arts[0].Labels) + require.EqualValues(t, "ddddd", arts[1].Digest) + require.Nil(t, arts[1].Labels) +} diff --git a/src/pkg/reg/model/policy.go b/src/pkg/reg/model/policy.go index c608ec3ea..60f8d6a28 100644 --- a/src/pkg/reg/model/policy.go +++ b/src/pkg/reg/model/policy.go @@ -24,12 +24,18 @@ const ( TriggerTypeManual = "manual" TriggerTypeScheduled = "scheduled" TriggerTypeEventBased = "event_based" + + // Matches [pattern] for tag (default) + Matches = "matches" + // Excludes [pattern] for tag + Excludes = "excludes" ) // Filter holds the info of the filter type Filter struct { - Type string `json:"type"` - Value interface{} `json:"value"` + Type string `json:"type"` + Value interface{} `json:"value"` + Decoration string `json:"decoration,omitempty"` } // Trigger holds info for a trigger diff --git a/src/pkg/reg/util/pattern_test.go b/src/pkg/reg/util/pattern_test.go index 11bff0f79..c5668f385 100644 --- a/src/pkg/reg/util/pattern_test.go +++ b/src/pkg/reg/util/pattern_test.go @@ -33,6 +33,11 @@ func TestMatch(t *testing.T) { str: "library", match: true, }, + { + pattern: "", + str: "", + match: true, + }, { pattern: "*", str: "library", diff --git a/src/server/v2.0/handler/replication.go b/src/server/v2.0/handler/replication.go index 81fc1deb1..d5a385915 100644 --- a/src/server/v2.0/handler/replication.go +++ b/src/server/v2.0/handler/replication.go @@ -85,8 +85,9 @@ func (r *replicationAPI) CreateReplicationPolicy(ctx context.Context, params ope if len(params.Policy.Filters) > 0 { for _, filter := range params.Policy.Filters { policy.Filters = append(policy.Filters, &model.Filter{ - Type: filter.Type, - Value: filter.Value, + Type: filter.Type, + Value: filter.Value, + Decoration: filter.Decoration, }) } } @@ -141,8 +142,9 @@ func (r *replicationAPI) UpdateReplicationPolicy(ctx context.Context, params ope if len(params.Policy.Filters) > 0 { for _, filter := range params.Policy.Filters { policy.Filters = append(policy.Filters, &model.Filter{ - Type: filter.Type, - Value: filter.Value, + Type: filter.Type, + Value: filter.Value, + Decoration: filter.Decoration, }) } } @@ -423,8 +425,9 @@ func convertReplicationPolicy(policy *repctlmodel.Policy) *models.ReplicationPol if len(policy.Filters) > 0 { for _, filter := range policy.Filters { p.Filters = append(p.Filters, &models.ReplicationFilter{ - Type: string(filter.Type), - Value: filter.Value, + Type: string(filter.Type), + Value: filter.Value, + Decoration: filter.Decoration, }) } }