Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-09-20 17:51:11 +02:00
Code Issues Projects Releases Wiki Activity

Make replication work with new artifact(phase 2)

Browse Source 
Provide the resource type filter for users to choose when replicating from harbor to other registries

Signed-off-by: Wenkai Yin <yinw@vmware.com>
This commit is contained in:
Wenkai Yin 2020-03-06 16:12:54 +08:00
parent c8ca6a5ccf
commit e237a686c4
7 changed files with 63 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/replication/adapter/harbor/adapter.go
View File

@ -98,6 +98,7 @@ func (a *adapter) Info() (*model.RegistryInfo, error) {
info := &model.RegistryInfo{
Type: model.RegistryTypeHarbor,
SupportedResourceTypes: []model.ResourceType{
model.ResourceTypeArtifact,
model.ResourceTypeImage,
},
SupportedResourceFilters: []*model.FilterStyle{

12
src/replication/adapter/harbor/adapter_test.go
View File

@ -45,9 +45,10 @@ func TestInfo(t *testing.T) {
assert.Equal(t, model.RegistryTypeHarbor, info.Type)
assert.Equal(t, 2, len(info.SupportedResourceFilters))
assert.Equal(t, 2, len(info.SupportedTriggers))
assert.Equal(t, 2, len(info.SupportedResourceTypes))
assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[1])
assert.Equal(t, 3, len(info.SupportedResourceTypes))
assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0])
assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1])
assert.Equal(t, model.ResourceTypeChart, info.SupportedResourceTypes[2])
server.Close()
// chart museum disabled
@ -69,8 +70,9 @@ func TestInfo(t *testing.T) {
assert.Equal(t, model.RegistryTypeHarbor, info.Type)
assert.Equal(t, 2, len(info.SupportedResourceFilters))
assert.Equal(t, 2, len(info.SupportedTriggers))
assert.Equal(t, 1, len(info.SupportedResourceTypes))
assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[0])
assert.Equal(t, 2, len(info.SupportedResourceTypes))
assert.Equal(t, model.ResourceTypeArtifact, info.SupportedResourceTypes[0])
assert.Equal(t, model.ResourceTypeImage, info.SupportedResourceTypes[1])
server.Close()
}

10
src/replication/filter/artifact.go
View File

@ -17,6 +17,7 @@ package filter
import (
"github.com/goharbor/harbor/src/replication/model"
"github.com/goharbor/harbor/src/replication/util"
"strings"
)
// DoFilterArtifacts filter the artifacts according to the filters
@ -42,6 +43,13 @@ func BuildArtifactFilters(filters []*model.Filter) (ArtifactFilters, error) {
f = &artifactTagFilter{
pattern: filter.Value.(string),
}
case model.FilterTypeResource:
v := filter.Value.(model.ResourceType)
if v != model.ResourceTypeArtifact && v != model.ResourceTypeChart {
f = &artifactTypeFilter{
types: []string{string(v)},
}
}
}
if f != nil {
fs = append(fs, f)
@ -81,7 +89,7 @@ func (a *artifactTypeFilter) Filter(artifacts []*model.Artifact) ([]*model.Artif
var result []*model.Artifact
for _, artifact := range artifacts {
for _, t := range a.types {
if artifact.Type == t {
if strings.ToLower(artifact.Type) == strings.ToLower(t) {
result = append(result, artifact)
continue
}

2
src/replication/model/policy.go
View File

@ -94,7 +94,7 @@ func (p *Policy) Valid(v *validation.Validation) {
}
if filter.Type == FilterTypeResource {
rt := ResourceType(value)
if !(rt == ResourceTypeImage || rt == ResourceTypeChart) {
if !(rt == ResourceTypeArtifact || rt == ResourceTypeImage || rt == ResourceTypeChart) {
v.SetError("filters", fmt.Sprintf("invalid resource filter: %s", value))
break
}

5
src/replication/model/resource.go
View File

@ -16,8 +16,9 @@ package model
// the resource type
const (
ResourceTypeImage ResourceType = "image"
ResourceTypeChart ResourceType = "chart"
ResourceTypeArtifact ResourceType = "artifact"
ResourceTypeImage ResourceType = "image"
ResourceTypeChart ResourceType = "chart"
)
// ResourceType represents the type of the resource

64
src/replication/operation/flow/stage.go
View File

@ -60,13 +60,10 @@ func initialize(policy *model.Policy) (adp.Adapter, adp.Adapter, error) {
// fetch resources from the source registry
func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resource, error) {
var resTypes []model.ResourceType
var filters []*model.Filter
for _, filter := range policy.Filters {
if filter.Type != model.FilterTypeResource {
filters = append(filters, filter)
continue
if filter.Type == model.FilterTypeResource {
resTypes = append(resTypes, filter.Value.(model.ResourceType))
}
resTypes = append(resTypes, filter.Value.(model.ResourceType))
}
if len(resTypes) == 0 {
info, err := adapter.Info()
@ -76,33 +73,42 @@ func fetchResources(adapter adp.Adapter, policy *model.Policy) ([]*model.Resourc
resTypes = append(resTypes, info.SupportedResourceTypes...)
}
resources := []*model.Resource{}
// convert the adapter to different interfaces according to its required resource types
for _, typ := range resTypes {
var res []*model.Resource
var err error
if typ == model.ResourceTypeImage {
// images
reg, ok := adapter.(adp.ImageRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ImageRegistry interface")
}
res, err = reg.FetchImages(filters)
} else if typ == model.ResourceTypeChart {
// charts
reg, ok := adapter.(adp.ChartRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface")
}
res, err = reg.FetchCharts(filters)
} else {
return nil, fmt.Errorf("unsupported resource type %s", typ)
fetchArtifact := false
fetchChart := false
for _, resType := range resTypes {
if resType == model.ResourceTypeChart {
fetchChart = true
continue
}
fetchArtifact = true
}
var resources []*model.Resource
// artifacts
if fetchArtifact {
reg, ok := adapter.(adp.ImageRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ImageRegistry interface")
}
res, err := reg.FetchImages(policy.Filters)
if err != nil {
return nil, fmt.Errorf("failed to fetch %s: %v", typ, err)
return nil, fmt.Errorf("failed to fetch artifacts: %v", err)
}
resources = append(resources, res...)
log.Debugf("fetch %s completed", typ)
log.Debug("fetch artifacts completed")
}
// charts
if fetchChart {
reg, ok := adapter.(adp.ChartRegistry)
if !ok {
return nil, fmt.Errorf("the adapter doesn't implement the ChartRegistry interface")
}
res, err := reg.FetchCharts(policy.Filters)
if err != nil {
return nil, fmt.Errorf("failed to fetch charts: %v", err)
}
resources = append(resources, res...)
log.Debug("fetch charts completed")
}
log.Debug("fetch resources from the source registry completed")
@ -290,7 +296,7 @@ func getResourceName(res *model.Resource) string {
n = len(meta.Vtags)
}
return fmt.Sprintf("%s [%d in total]", meta.Repository.Name, n)
return fmt.Sprintf("%s [%d artifact(s) in total]", meta.Repository.Name, n)
}
// repository:c namespace:n -> n/c

10
src/replication/transfer/image/transfer.go
View File

@ -16,6 +16,8 @@ package image
import (
"errors"
"github.com/docker/distribution/manifest/manifestlist"
"github.com/docker/distribution/manifest/schema1"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
"strings"
@ -214,9 +216,11 @@ func (t *transfer) copyImage(srcRepo, srcRef, dstRepo, dstRef string, override b
func (t *transfer) copyContent(content distribution.Descriptor, srcRepo, dstRepo string) error {
digest := content.Digest.String()
switch content.MediaType {
// when the media type of pulled manifest is manifest list,
// the contents it contains are a few manifests
case v1.MediaTypeImageManifest, schema2.MediaTypeManifest:
// when the media type of pulled manifest is index,
// the contents it contains are a few manifests/indexes
case v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList,
v1.MediaTypeImageManifest, schema2.MediaTypeManifest,
schema1.MediaTypeSignedManifest:
// as using digest as the reference, so set the override to true directly
return t.copyImage(srcRepo, digest, dstRepo, digest, true)
// handle foreign layer