Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-22 18:25:56 +01:00
Code Issues Projects Releases Wiki Activity

Merge pull request #15344 from bitsf/replication_policy_exclude

Browse Source 
feat(replication) add exclude decoration in policy
This commit is contained in:
Steven Zou 2021-08-19 14:26:17 +08:00 committed by GitHub
commit f3a875abd7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 307 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/v2.0/swagger.yaml
View File

@ -6477,6 +6477,9 @@ definitions:
value:
type: object
description: 'The value of replication policy filter.'
decoration:
type: string
description: 'matches or excludes the result'
RegistryCredential:
type: object
properties:

48
src/controller/replication/model/model.go
View File

@ -79,37 +79,9 @@ func (p *Policy) Validate() error {
}
// valid the filters
for _, filter := range p.Filters {
switch filter.Type {
case model.FilterTypeResource, model.FilterTypeName, model.FilterTypeTag:
value, ok := filter.Value.(string)
if !ok {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("the type of filter value isn't string")
}
if filter.Type == model.FilterTypeResource {
rt := value
if !(rt == model.ResourceTypeArtifact || rt == model.ResourceTypeImage || rt == model.ResourceTypeChart) {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("invalid resource filter: %s", value)
}
}
case model.FilterTypeLabel:
labels, ok := filter.Value.([]interface{})
if !ok {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("the type of label filter value isn't string slice")
}
for _, label := range labels {
_, ok := label.(string)
if !ok {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("the type of label filter value isn't string slice")
}
}
default:
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("invalid filter type")
for _, f := range p.Filters {
if err := f.Validate(); err != nil {
return err
}
}
@ -229,10 +201,11 @@ func (p *Policy) To() (*replicationmodel.Policy, error) {
}
type filter struct {
Type string `json:"type"`
Value interface{} `json:"value"`
Kind string `json:"kind"`
Pattern string `json:"pattern"`
Type string `json:"type"`
Value interface{} `json:"value"`
Decoration string `json:"decoration"`
Kind string `json:"kind"`
Pattern string `json:"pattern"`
}
type trigger struct {
@ -260,8 +233,9 @@ func parseFilters(str string) ([]*model.Filter, error) {
filters := []*model.Filter{}
for _, item := range items {
filter := &model.Filter{
Type: item.Type,
Value: item.Value,
Type: item.Type,
Value: item.Value,
Decoration: item.Decoration,
}
// keep backwards compatibility
if len(filter.Type) == 0 {

37
src/pkg/reg/adapter/aliacr/adapter.go
View File

@ -18,6 +18,7 @@ import (
"github.com/goharbor/harbor/src/lib/log"
adp "github.com/goharbor/harbor/src/pkg/reg/adapter"
"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
"github.com/goharbor/harbor/src/pkg/reg/filter"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
"github.com/goharbor/harbor/src/pkg/registry/auth/bearer"
@ -236,12 +237,9 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Re
// get filter pattern
var repoPattern string
var tagsPattern string
for _, filter := range filters {
if filter.Type == model.FilterTypeName {
repoPattern = filter.Value.(string)
}
if filter.Type == model.FilterTypeTag {
tagsPattern = filter.Value.(string)
for _, f := range filters {
if f.Type == model.FilterTypeName {
repoPattern = f.Value.(string)
}
}
var namespacePattern = strings.Split(repoPattern, "/")[0]
@ -295,23 +293,18 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Re
return fmt.Errorf("list tags for repo '%s' error: %v", repo.RepoName, err)
}
var filterTags []string
if tagsPattern != "" {
for _, tag := range tags {
var ok bool
ok, err = util.Match(tagsPattern, tag)
if err != nil {
return fmt.Errorf("match tag '%s' error: %v", tag, err)
}
if ok {
filterTags = append(filterTags, tag)
}
}
} else {
filterTags = tags
var artifacts []*model.Artifact
for _, tag := range tags {
artifacts = append(artifacts, &model.Artifact{
Tags: []string{tag},
})
}
filterArtifacts, err := filter.DoFilterArtifacts(artifacts, filters)
if err != nil {
return err
}
if len(filterTags) > 0 {
if len(filterArtifacts) > 0 {
rawResources[index] = &model.Resource{
Type: model.ResourceTypeImage,
Registry: a.registry,
@ -319,7 +312,7 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) (resources []*model.Re
Repository: &model.Repository{
Name: filepath.Join(repo.RepoNamespace, repo.RepoName),
},
Vtags: filterTags,
Artifacts: filterArtifacts,
},
}
}

29
src/pkg/reg/adapter/dockerhub/adapter.go
View File

@ -13,6 +13,7 @@ import (
"github.com/goharbor/harbor/src/lib/log"
adp "github.com/goharbor/harbor/src/pkg/reg/adapter"
"github.com/goharbor/harbor/src/pkg/reg/adapter/native"
"github.com/goharbor/harbor/src/pkg/reg/filter"
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/goharbor/harbor/src/pkg/reg/util"
)
@ -245,10 +246,6 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
if err != nil {
return nil, err
}
tagFilter, err := a.getStringFilterValue(model.FilterTypeTag, filters)
if err != nil {
return nil, err
}
namespaces, err := a.listCandidateNamespaces(nameFilter)
if err != nil {
@ -305,17 +302,6 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
return fmt.Errorf("get tags for repo '%s/%s' from DockerHub error: %v", repo.Namespace, repo.Name, err)
}
for _, t := range pageTags.Tags {
// If tag filter set, skip tags that don't match the filter pattern.
if len(tagFilter) != 0 {
m, err := util.Match(tagFilter, t.Name)
if err != nil {
return fmt.Errorf("match tag name '%s' against pattern '%s' error: %v", t.Name, tagFilter, err)
}
if !m {
continue
}
}
tags = append(tags, t.Name)
}
@ -325,6 +311,17 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
page++
}
var artifacts []*model.Artifact
for _, tag := range tags {
artifacts = append(artifacts, &model.Artifact{
Tags: []string{tag},
})
}
filterArtifacts, err := filter.DoFilterArtifacts(artifacts, filters)
if err != nil {
return err
}
if len(tags) > 0 {
rawResources[index] = &model.Resource{
Type: model.ResourceTypeImage,
@ -333,7 +330,7 @@ func (a *adapter) FetchArtifacts(filters []*model.Filter) ([]*model.Resource, er
Repository: &model.Repository{
Name: name,
},
Vtags: tags,
Artifacts: filterArtifacts,
},
}
}

43
src/pkg/reg/filter/artifact.go
View File

@ -37,17 +37,19 @@ func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
switch filter.Type {
case model.FilterTypeLabel:
f = &artifactLabelFilter{
labels: filter.Value.([]string),
labels: filter.Value.([]string),
decoration: filter.Decoration,
}
case model.FilterTypeTag:
f = &artifactTagFilter{
pattern: filter.Value.(string),
pattern: filter.Value.(string),
decoration: filter.Decoration,
}
case model.FilterTypeResource:
v := filter.Value.(string)
if v != model.ResourceTypeArtifact && v != model.ResourceTypeChart {
f = &artifactTypeFilter{
types: []string{string(v)},
types: []string{v},
}
}
}
@ -102,6 +104,8 @@ func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artif
// in the filter is the valid one
type artifactLabelFilter struct {
labels []string
// "matches", "excludes"
decoration string
}
func (a *artifactLabelFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
@ -122,8 +126,14 @@ func (a *artifactLabelFilter) Filter(artifacts []*model.Artifact) ([]*model.Arti
}
}
// add the artifact to the result list if it contains all labels defined for the filter
if match {
result = append(result, artifact)
if a.decoration == model.Excludes {
if !match {
result = append(result, artifact)
}
} else {
if match {
result = append(result, artifact)
}
}
}
return result, nil
@ -147,6 +157,8 @@ func (a *artifactTaggedFilter) Filter(artifacts []*model.Artifact) ([]*model.Art
type artifactTagFilter struct {
pattern string
// "matches", "excludes"
decoration string
}
func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifact, error) {
@ -161,8 +173,14 @@ func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifa
if err != nil {
return nil, err
}
if match {
result = append(result, artifact)
if a.decoration == model.Excludes {
if !match {
result = append(result, artifact)
}
} else {
if match {
result = append(result, artifact)
}
}
continue
}
@ -174,9 +192,14 @@ func (a *artifactTagFilter) Filter(artifacts []*model.Artifact) ([]*model.Artifa
if err != nil {
return nil, err
}
if match {
tags = append(tags, tag)
continue
if a.decoration == model.Excludes {
if !match {
tags = append(tags, tag)
}
} else {
if match {
tags = append(tags, tag)
}
}
}
if len(tags) == 0 {

161
src/pkg/reg/filter/resource_test.go Normal file
View File

@ -0,0 +1,161 @@
package filter
import (
"github.com/goharbor/harbor/src/pkg/reg/model"
"github.com/stretchr/testify/require"
"testing"
)
func TestArtifactTagFilters(t *testing.T) {
var artifacts = []*model.Artifact{
{
Type: model.ResourceTypeArtifact,
Digest: "aaaaa",
Tags: []string{
"test1",
"test2",
"harbor1",
},
},
{
Type: model.ResourceTypeArtifact,
Digest: "bbbbb",
Tags: []string{
"test3",
"harbor2",
},
},
{
Type: model.ResourceTypeArtifact,
Digest: "ccccc",
Tags: []string{
"harbor3",
},
},
{
Type: model.ResourceTypeArtifact,
Digest: "ddddd",
},
}
var filters = []*model.Filter{
{
Type: model.FilterTypeTag,
Value: "test*",
},
}
artFilters, err := BuildArtifactFilters(filters)
require.Nil(t, err)
arts, err := artFilters.Filter(artifacts)
require.Nil(t, err)
require.Equal(t, 2, len(arts))
require.EqualValues(t, "aaaaa", arts[0].Digest)
require.EqualValues(t, []string{"test1", "test2"}, arts[0].Tags)
require.EqualValues(t, "bbbbb", arts[1].Digest)
require.EqualValues(t, []string{"test3"}, arts[1].Tags)
filters = []*model.Filter{
{
Type: model.FilterTypeTag,
Value: "test*",
Decoration: model.Excludes,
},
}
artFilters, err = BuildArtifactFilters(filters)
require.Nil(t, err)
arts, err = artFilters.Filter(artifacts)
require.Nil(t, err)
require.Equal(t, 4, len(arts))
require.EqualValues(t, "aaaaa", arts[0].Digest)
require.EqualValues(t, []string{"harbor1"}, arts[0].Tags)
require.EqualValues(t, "bbbbb", arts[1].Digest)
require.EqualValues(t, []string{"harbor2"}, arts[1].Tags)
require.EqualValues(t, "ccccc", arts[2].Digest)
require.EqualValues(t, []string{"harbor3"}, arts[2].Tags)
require.EqualValues(t, "ddddd", arts[3].Digest)
require.Nil(t, arts[3].Tags)
}
func TestArtifactLabelFilters(t *testing.T) {
var artifacts = []*model.Artifact{
{
Type: model.ResourceTypeArtifact,
Digest: "aaaaa",
Tags: []string{
"test1",
"test2",
"harbor1",
},
Labels: []string{
"label1",
},
},
{
Type: model.ResourceTypeArtifact,
Digest: "bbbbb",
Tags: []string{
"test3",
"harbor2",
},
Labels: []string{
"label1",
"label2",
},
},
{
Type: model.ResourceTypeArtifact,
Digest: "ccccc",
Tags: []string{
"harbor3",
},
Labels: []string{
"label3",
},
},
{
Type: model.ResourceTypeArtifact,
Digest: "ddddd",
},
}
var filters = []*model.Filter{
{
Type: model.FilterTypeLabel,
Value: []string{"label1"},
},
}
artFilters, err := BuildArtifactFilters(filters)
require.Nil(t, err)
arts, err := artFilters.Filter(artifacts)
require.Nil(t, err)
require.Equal(t, 2, len(arts))
require.EqualValues(t, "aaaaa", arts[0].Digest)
require.EqualValues(t, []string{"label1"}, arts[0].Labels)
require.EqualValues(t, "bbbbb", arts[1].Digest)
require.EqualValues(t, []string{"label1", "label2"}, arts[1].Labels)
filters = []*model.Filter{
{
Type: model.FilterTypeLabel,
Value: []string{"label1"},
Decoration: model.Excludes,
},
}
artFilters, err = BuildArtifactFilters(filters)
require.Nil(t, err)
arts, err = artFilters.Filter(artifacts)
require.Nil(t, err)
require.Equal(t, 2, len(arts))
require.EqualValues(t, "ccccc", arts[0].Digest)
require.EqualValues(t, []string{"label3"}, arts[0].Labels)
require.EqualValues(t, "ddddd", arts[1].Digest)
require.Nil(t, arts[1].Labels)
}

59
src/pkg/reg/model/policy.go
View File

@ -14,6 +14,8 @@
package model
import "github.com/goharbor/harbor/src/lib/errors"
// const definition
const (
FilterTypeResource = "resource"
@ -24,12 +26,65 @@ const (
TriggerTypeManual = "manual"
TriggerTypeScheduled = "scheduled"
TriggerTypeEventBased = "event_based"
// Matches [pattern] for tag (default)
Matches = "matches"
// Excludes [pattern] for tag
Excludes = "excludes"
)
// Filter holds the info of the filter
type Filter struct {
Type string `json:"type"`
Value interface{} `json:"value"`
Type string `json:"type"`
Value interface{} `json:"value"`
Decoration string `json:"decoration,omitempty"`
}
func (f *Filter) Validate() error {
switch f.Type {
case FilterTypeResource, FilterTypeName, FilterTypeTag:
value, ok := f.Value.(string)
if !ok {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("the type of filter value isn't string")
}
if f.Type == FilterTypeResource {
rt := value
if !(rt == ResourceTypeArtifact || rt == ResourceTypeImage || rt == ResourceTypeChart) {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("invalid resource filter: %s", value)
}
}
if f.Type == FilterTypeName || f.Type == FilterTypeResource {
if f.Decoration != "" {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("only tag and label filter support decoration")
}
}
case FilterTypeLabel:
labels, ok := f.Value.([]interface{})
if !ok {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("the type of label filter value isn't string slice")
}
for _, label := range labels {
_, ok := label.(string)
if !ok {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("the type of label filter value isn't string slice")
}
}
default:
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("invalid filter type")
}
if f.Decoration != "" && f.Decoration != Matches && f.Decoration != Excludes {
return errors.New(nil).WithCode(errors.BadRequestCode).
WithMessage("invalid filter decoration, :%s", f.Decoration)
}
return nil
}
// Trigger holds info for a trigger

5
src/pkg/reg/util/pattern_test.go
View File

@ -33,6 +33,11 @@ func TestMatch(t *testing.T) {
str: "library",
match: true,
},
{
pattern: "",
str: "",
match: true,
},
{
pattern: "*",
str: "library",

15
src/server/v2.0/handler/replication.go
View File

@ -85,8 +85,9 @@ func (r *replicationAPI) CreateReplicationPolicy(ctx context.Context, params ope
if len(params.Policy.Filters) > 0 {
for _, filter := range params.Policy.Filters {
policy.Filters = append(policy.Filters, &model.Filter{
Type: filter.Type,
Value: filter.Value,
Type: filter.Type,
Value: filter.Value,
Decoration: filter.Decoration,
})
}
}
@ -141,8 +142,9 @@ func (r *replicationAPI) UpdateReplicationPolicy(ctx context.Context, params ope
if len(params.Policy.Filters) > 0 {
for _, filter := range params.Policy.Filters {
policy.Filters = append(policy.Filters, &model.Filter{
Type: filter.Type,
Value: filter.Value,
Type: filter.Type,
Value: filter.Value,
Decoration: filter.Decoration,
})
}
}
@ -423,8 +425,9 @@ func convertReplicationPolicy(policy *repctlmodel.Policy) *models.ReplicationPol
if len(policy.Filters) > 0 {
for _, filter := range policy.Filters {
p.Filters = append(p.Filters, &models.ReplicationFilter{
Type: string(filter.Type),
Value: filter.Value,
Type: string(filter.Type),
Value: filter.Value,
Decoration: filter.Decoration,
})
}
}