Fix replication filter bug

This commit fixes the bug of replication filter, see #13593 for more detail
Fixes #13593

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2021-01-05 21:44:15 +08:00
parent d0152cb446
commit d474750e9f
7 changed files with 57 additions and 136 deletions

View File

@ -51,15 +51,13 @@ func (c *copyFlow) Run(ctx context.Context) error {
if err != nil { if err != nil {
return err return err
} }
var srcResources []*model.Resource srcResources := c.resources
if len(c.resources) > 0 { if len(srcResources) == 0 {
srcResources, err = filterResources(c.resources, c.policy.Filters)
} else {
srcResources, err = fetchResources(srcAdapter, c.policy) srcResources, err = fetchResources(srcAdapter, c.policy)
}
if err != nil { if err != nil {
return err return err
} }
}
isStopped, err := c.isExecutionStopped(ctx) isStopped, err := c.isExecutionStopped(ctx)
if err != nil { if err != nil {

View File

@ -19,7 +19,6 @@ import (
"encoding/json" "encoding/json"
"github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/task" "github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
) )
@ -45,20 +44,7 @@ func NewDeletionFlow(executionID int64, policy *model.Policy, resources ...*mode
} }
func (d *deletionFlow) Run(ctx context.Context) error { func (d *deletionFlow) Run(ctx context.Context) error {
logger := log.GetLogger(ctx) srcResources := assembleSourceResources(d.resources, d.policy)
srcResources, err := filterResources(d.resources, d.policy.Filters)
if err != nil {
return err
}
if len(srcResources) == 0 {
// no candidates, mark the execution as done directly
if err := d.executionMgr.MarkDone(ctx, d.executionID, "no resources need to be replicated"); err != nil {
logger.Errorf("failed to mark done for the execution %d: %v", d.executionID, err)
}
return nil
}
srcResources = assembleSourceResources(srcResources, d.policy)
dstResources := assembleDestinationResources(srcResources, d.policy) dstResources := assembleDestinationResources(srcResources, d.policy)
return d.createTasks(ctx, srcResources, dstResources) return d.createTasks(ctx, srcResources, dstResources)

View File

@ -19,7 +19,6 @@ import (
"github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/log"
adp "github.com/goharbor/harbor/src/replication/adapter" adp "github.com/goharbor/harbor/src/replication/adapter"
"github.com/goharbor/harbor/src/replication/filter"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util" "github.com/goharbor/harbor/src/replication/util"
) )
@ -110,16 +109,6 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc
return resources, nil return resources, nil
} }
// apply the filters to the resources and returns the filtered resources
func filterResources(resources []*model.Resource, filters []*model.Filter) ([]*model.Resource, error) {
resources, err := filter.DoFilterResources(resources, filters)
if err != nil {
return nil, err
}
log.Debug("filter resources completed")
return resources, nil
}
// assemble the source resources by filling the registry information // assemble the source resources by filling the registry information
func assembleSourceResources(resources []*model.Resource, func assembleSourceResources(resources []*model.Resource,
policy *model.Policy) []*model.Resource { policy *model.Policy) []*model.Resource {

View File

@ -67,83 +67,6 @@ func (s *stageTestSuite) TestFetchResources() {
adapter.AssertExpectations(s.T()) adapter.AssertExpectations(s.T())
} }
func (s *stageTestSuite) TestFilterResources() {
resources := []*model.Resource{
{
Type: model.ResourceTypeImage,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/hello-world",
},
Artifacts: []*model.Artifact{
{
Tags: []string{"latest"},
},
},
},
Deleted: true,
Override: true,
},
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/harbor",
},
Artifacts: []*model.Artifact{
{
Tags: []string{"0.2.0"},
},
{
Tags: []string{"0.3.0"},
},
},
},
Deleted: true,
Override: true,
},
{
Type: model.ResourceTypeChart,
Metadata: &model.ResourceMetadata{
Repository: &model.Repository{
Name: "library/mysql",
},
Artifacts: []*model.Artifact{
{
Tags: []string{"1.0"},
},
},
},
Deleted: true,
Override: true,
},
}
filters := []*model.Filter{
{
Type: model.FilterTypeResource,
Value: model.ResourceTypeChart,
},
{
Type: model.FilterTypeName,
Value: "library/*",
},
{
Type: model.FilterTypeName,
Value: "library/harbor",
},
{
Type: model.FilterTypeTag,
Value: "0.2.?",
},
}
res, err := filterResources(resources, filters)
s.Require().Nil(err)
s.Len(res, 1)
s.Equal("library/harbor", res[0].Metadata.Repository.Name)
s.Equal(1, len(res[0].Metadata.Artifacts))
s.Equal("0.2.0", res[0].Metadata.Artifacts[0].Tags[0])
}
func (s *stageTestSuite) TestAssembleSourceResources() { func (s *stageTestSuite) TestAssembleSourceResources() {
resources := []*model.Resource{ resources := []*model.Resource{
{ {

View File

@ -17,17 +17,17 @@ package event
import ( import (
"errors" "errors"
"fmt" "fmt"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/task"
commonthttp "github.com/goharbor/harbor/src/common/http" commonthttp "github.com/goharbor/harbor/src/common/http"
"github.com/goharbor/harbor/src/controller/replication" "github.com/goharbor/harbor/src/controller/replication"
"github.com/goharbor/harbor/src/lib/log" "github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/task"
"github.com/goharbor/harbor/src/replication/config" "github.com/goharbor/harbor/src/replication/config"
"github.com/goharbor/harbor/src/replication/filter"
"github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/policy" "github.com/goharbor/harbor/src/replication/policy"
"github.com/goharbor/harbor/src/replication/registry" "github.com/goharbor/harbor/src/replication/registry"
"github.com/goharbor/harbor/src/replication/util"
) )
// Handler is the handler to handle event // Handler is the handler to handle event
@ -115,39 +115,21 @@ func (h *handler) getRelatedPolicies(resource *model.Resource) ([]*model.Policy,
if resource.Deleted && !policy.Deletion { if resource.Deleted && !policy.Deletion {
continue continue
} }
// doesn't match the name filter
m, err := match(policy.Filters, resource) resources, err := filter.DoFilterResources([]*model.Resource{resource}, policy.Filters)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !m { // doesn't match the filters
if len(resources) == 0 {
continue continue
} }
result = append(result, policy) result = append(result, policy)
} }
return result, nil return result, nil
} }
// TODO unify the match logic with other?
func match(filters []*model.Filter, resource *model.Resource) (bool, error) {
match := true
repository := resource.Metadata.Repository.Name
for _, filter := range filters {
if filter.Type != model.FilterTypeName {
continue
}
m, err := util.Match(filter.Value.(string), repository)
if err != nil {
return false, err
}
if !m {
match = false
break
}
}
return match, nil
}
// PopulateRegistries populates the source registry and destination registry properties for policy // PopulateRegistries populates the source registry and destination registry properties for policy
func PopulateRegistries(registryMgr registry.Manager, policy *model.Policy) error { func PopulateRegistries(registryMgr registry.Manager, policy *model.Policy) error {
if policy == nil { if policy == nil {

View File

@ -190,6 +190,13 @@ func TestGetRelatedPolicies(t *testing.T) {
Repository: &model.Repository{ Repository: &model.Repository{
Name: "library/hello-world", Name: "library/hello-world",
}, },
Artifacts: []*model.Artifact{
{
Type: "image",
Digest: "sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042",
Tags: []string{"latest"},
},
},
}, },
}) })
require.Nil(t, err) require.Nil(t, err)
@ -202,6 +209,13 @@ func TestGetRelatedPolicies(t *testing.T) {
Repository: &model.Repository{ Repository: &model.Repository{
Name: "library/hello-world", Name: "library/hello-world",
}, },
Artifacts: []*model.Artifact{
{
Type: "image",
Digest: "sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042",
Tags: []string{"latest"},
},
},
}, },
Deleted: true, Deleted: true,
}) })

View File

@ -56,5 +56,34 @@ func DoFilterResources(resources []*model.Resource, filters []*model.Filter) ([]
Override: resource.Override, Override: resource.Override,
}) })
} }
// remove this after we deprecate chart museum
return filterByResourceType(result, filters)
}
// After we deprecated chart museum, the resource types model.ResourceTypeArtifact and model.ResourceTypeChart
// are useless, this function should be removed as well
func filterByResourceType(resources []*model.Resource, filters []*model.Filter) ([]*model.Resource, error) {
var resourceType model.ResourceType
for _, filter := range filters {
if filter.Type == model.FilterTypeResource {
// model.ResourceTypeImage is handled by artifact filters in function "DoFilterResources"
if filter.Value.(model.ResourceType) == model.ResourceTypeArtifact || filter.Value.(model.ResourceType) == model.ResourceTypeChart {
resourceType = filter.Value.(model.ResourceType)
}
break
}
}
// no resource type, return the candidates directly
if len(resourceType) == 0 {
return resources, nil
}
var result []*model.Resource
for _, resource := range resources {
if resource.Type == resourceType {
result = append(result, resource)
}
}
return result, nil return result, nil
} }