diff --git a/src/replication/adapter/harbor/adapter.go b/src/replication/adapter/harbor/adapter.go index 03f60e751b..d265207677 100644 --- a/src/replication/adapter/harbor/adapter.go +++ b/src/replication/adapter/harbor/adapter.go @@ -98,6 +98,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) { info := &model.RegistryInfo{ Type: model.RegistryTypeHarbor, SupportedResourceTypes: []model.ResourceType{ + model.ResourceTypeArtifact, model.ResourceTypeImage, }, SupportedResourceFilters: []*model.FilterStyle{ diff --git a/src/replication/adapter/harbor/adapter_test.go b/src/replication/adapter/harbor/adapter_test.go index a51f2653d5..f553bdb2d7 100644 --- a/src/replication/adapter/harbor/adapter_test.go +++ b/src/replication/adapter/harbor/adapter_test.go @@ -45,9 +45,10 @@ func TestInfo(t *testing.T) { assert.Equal(t, model.RegistryTypeHarbor, info.Type) assert.Equal(t, 2, len(info.SupportedResourceFilters)) assert.Equal(t, 2, len(info.SupportedTriggers)) - assert.Equal(t, 2, len(info.SupportedResourceTypes)) - assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0]) - assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[1]) + assert.Equal(t, 3, len(info.SupportedResourceTypes)) + assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0]) + assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1]) + assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[2]) server.Close() // chart museum disabled @@ -69,8 +70,9 @@ func TestInfo(t *testing.T) { assert.Equal(t, model.RegistryTypeHarbor, info.Type) assert.Equal(t, 2, len(info.SupportedResourceFilters)) assert.Equal(t, 2, len(info.SupportedTriggers)) - assert.Equal(t, 1, len(info.SupportedResourceTypes)) - assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0]) + assert.Equal(t, 2, len(info.SupportedResourceTypes)) + assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0]) + assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1]) server.Close() } diff --git a/src/replication/filter/artifact.go b/src/replication/filter/artifact.go index 86e882468d..cc88279c88 100644 --- a/src/replication/filter/artifact.go +++ b/src/replication/filter/artifact.go @@ -17,6 +17,7 @@ package filter import ( "github.com/goharbor/harbor/src/replication/model" "github.com/goharbor/harbor/src/replication/util" + "strings" ) // DoFilterArtifacts filter the artifacts according to the filters @@ -42,6 +43,13 @@ func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) { f = &artifactTagFilter{ pattern: filter.Value.(string), } + case model.FilterTypeResource: + v := filter.Value.(model.ResourceType) + if v != model.ResourceTypeArtifact && v != model.ResourceTypeChart { + f = &artifactTypeFilter{ + types: []string{string(v)}, + } + } } if f != nil { fs = append(fs, f) @@ -81,7 +89,7 @@ func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artif var result []*model.Artifact for _, artifact := range artifacts { for _, t := range a.types { - if artifact.Type == t { + if strings.ToLower(artifact.Type) == strings.ToLower(t) { result = append(result, artifact) continue } diff --git a/src/replication/model/policy.go b/src/replication/model/policy.go index b7b14d3818..6036357781 100644 --- a/src/replication/model/policy.go +++ b/src/replication/model/policy.go @@ -94,7 +94,7 @@ func (p *Policy) Valid(v *validation.Validation) { } if filter.Type == FilterTypeResource { rt := ResourceType(value) - if !(rt == ResourceTypeImage || rt == ResourceTypeChart) { + if !(rt == ResourceTypeArtifact || rt == ResourceTypeImage || rt == ResourceTypeChart) { v.SetError("filters", fmt.Sprintf("invalid resource filter: %s", value)) break } diff --git a/src/replication/model/resource.go b/src/replication/model/resource.go index da747120d4..e608d8d2dc 100644 --- a/src/replication/model/resource.go +++ b/src/replication/model/resource.go @@ -16,8 +16,9 @@ package model // the resource type const ( - ResourceTypeImage ResourceType = "image" - ResourceTypeChart ResourceType = "chart" + ResourceTypeArtifact ResourceType = "artifact" + ResourceTypeImage ResourceType = "image" + ResourceTypeChart ResourceType = "chart" ) // ResourceType represents the type of the resource diff --git a/src/replication/operation/flow/stage.go b/src/replication/operation/flow/stage.go index 449a1d9fd4..d8670cdbe1 100644 --- a/src/replication/operation/flow/stage.go +++ b/src/replication/operation/flow/stage.go @@ -60,13 +60,10 @@ func initialize(policy *model.Policy) (adp.Adapter, adp.Adapter, error) { // fetch resources from the source registry func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resource, error) { var resTypes []model.ResourceType - var filters []*model.Filter for _, filter := range policy.Filters { - if filter.Type != model.FilterTypeResource { - filters = append(filters, filter) - continue + if filter.Type == model.FilterTypeResource { + resTypes = append(resTypes, filter.Value.(model.ResourceType)) } - resTypes = append(resTypes, filter.Value.(model.ResourceType)) } if len(resTypes) == 0 { info, err := adapter.Info() @@ -76,33 +73,42 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc resTypes = append(resTypes, info.SupportedResourceTypes...) } - resources := []*model.Resource{} - // convert the adapter to different interfaces according to its required resource types - for _, typ := range resTypes { - var res []*model.Resource - var err error - if typ == model.ResourceTypeImage { - // images - reg, ok := adapter.(adp.ImageRegistry) - if !ok { - return nil, fmt.Errorf("the adapter doesn't implement the ImageRegistry interface") - } - res, err = reg.FetchImages(filters) - } else if typ == model.ResourceTypeChart { - // charts - reg, ok := adapter.(adp.ChartRegistry) - if !ok { - return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface") - } - res, err = reg.FetchCharts(filters) - } else { - return nil, fmt.Errorf("unsupported resource type %s", typ) + fetchArtifact := false + fetchChart := false + for _, resType := range resTypes { + if resType == model.ResourceTypeChart { + fetchChart = true + continue } + fetchArtifact = true + } + + var resources []*model.Resource + // artifacts + if fetchArtifact { + reg, ok := adapter.(adp.ImageRegistry) + if !ok { + return nil, fmt.Errorf("the adapter doesn't implement the ImageRegistry interface") + } + res, err := reg.FetchImages(policy.Filters) if err != nil { - return nil, fmt.Errorf("failed to fetch %s: %v", typ, err) + return nil, fmt.Errorf("failed to fetch artifacts: %v", err) } resources = append(resources, res...) - log.Debugf("fetch %s completed", typ) + log.Debug("fetch artifacts completed") + } + // charts + if fetchChart { + reg, ok := adapter.(adp.ChartRegistry) + if !ok { + return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface") + } + res, err := reg.FetchCharts(policy.Filters) + if err != nil { + return nil, fmt.Errorf("failed to fetch charts: %v", err) + } + resources = append(resources, res...) + log.Debug("fetch charts completed") } log.Debug("fetch resources from the source registry completed") @@ -290,7 +296,7 @@ func getResourceName(res *model.Resource) string { n = len(meta.Vtags) } - return fmt.Sprintf("%s [%d in total]", meta.Repository.Name, n) + return fmt.Sprintf("%s [%d artifact(s) in total]", meta.Repository.Name, n) } // repository:c namespace:n -> n/c diff --git a/src/replication/transfer/image/transfer.go b/src/replication/transfer/image/transfer.go index a8dbc4d8dc..e139c7e8a4 100644 --- a/src/replication/transfer/image/transfer.go +++ b/src/replication/transfer/image/transfer.go @@ -16,6 +16,8 @@ package image import ( "errors" + "github.com/docker/distribution/manifest/manifestlist" + "github.com/docker/distribution/manifest/schema1" v1 "github.com/opencontainers/image-spec/specs-go/v1" "strings" @@ -214,9 +216,11 @@ func (t *transfer) copyImage(srcRepo, srcRef, dstRepo, dstRef string, override b func (t *transfer) copyContent(content distribution.Descriptor, srcRepo, dstRepo string) error { digest := content.Digest.String() switch content.MediaType { - // when the media type of pulled manifest is manifest list, - // the contents it contains are a few manifests - case v1.MediaTypeImageManifest, schema2.MediaTypeManifest: + // when the media type of pulled manifest is index, + // the contents it contains are a few manifests/indexes + case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList, + v1.MediaTypeImageManifest, schema2.MediaTypeManifest, + schema1.MediaTypeSignedManifest: // as using digest as the reference, so set the override to true directly return t.copyImage(srcRepo, digest, dstRepo, digest, true) // handle foreign layer